[ May 1 05:08:03 Enabled. ] [ May 1 05:08:03 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ May 1 05:08:03 Method "start" exited with status 0. ] [ May 1 05:08:03 Rereading configuration. ] [ May 1 05:08:03 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:03.415431537Z","hostname":"buskin","pid":1590,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:03.415601301Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:03.471481635Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:03.471536896Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:03.590060327Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:03.599237768Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:03.607537511Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.21485217Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:9fb2::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.357436502Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:119"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.357468303Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:131"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.357530174Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.357662367Z","hostname":"buskin","pid":1590,"component":"DumpSetup-archival","file":"sled-agent/src/dump_setup.rs:291"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.357680677Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:162","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.357686158Z","hostname":"buskin","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.357731219Z","hostname":"buskin","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:625"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.360822113Z","hostname":"buskin","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:04.360835493Z","hostname":"buskin","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:674"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.360851673Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.360857814Z","hostname":"buskin","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:733"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.360876094Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.360947935Z","hostname":"buskin","pid":1590,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.361025557Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.361043828Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.361068098Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.373438106Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.373479817Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.473678025Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: 57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.683655322Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.683688923Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.683693723Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.683705003Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.683710613Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.683727824Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.683732514Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.692590908Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.692620409Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"07066414-66e9-4cbf-a342-0d2cd3c1d2ca (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.781480841Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: 07066414-66e9-4cbf-a342-0d2cd3c1d2ca (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989313843Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989361834Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 07066414-66e9-4cbf-a342-0d2cd3c1d2ca (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989404465Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989410795Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989418465Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989464806Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 07066414-66e9-4cbf-a342-0d2cd3c1d2ca (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989484266Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989489927Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989525897Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989551888Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 07066414-66e9-4cbf-a342-0d2cd3c1d2ca (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989584778Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989589369Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989595699Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_3.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989600469Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989604579Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 07066414-66e9-4cbf-a342-0d2cd3c1d2ca (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989622779Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989632859Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_4.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989638279Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.98964238Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.9896485Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 07066414-66e9-4cbf-a342-0d2cd3c1d2ca (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.98966731Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_5.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.9896835Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989688301Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989693811Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 07066414-66e9-4cbf-a342-0d2cd3c1d2ca (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989715141Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989721151Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989727452Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989732232Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 07066414-66e9-4cbf-a342-0d2cd3c1d2ca (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989754302Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989759402Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989772492Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989777713Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 07066414-66e9-4cbf-a342-0d2cd3c1d2ca (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989801353Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989806673Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989810533Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989816793Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 07066414-66e9-4cbf-a342-0d2cd3c1d2ca (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989852124Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:91"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989863954Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:93"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989868274Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989872124Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 07066414-66e9-4cbf-a342-0d2cd3c1d2ca (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989900035Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"No ledger in /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989939676Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989957656Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989966346Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989976897Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.989983487Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.990002227Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.990013958Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:04.990024728Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.000510516Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:711"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.000556417Z","hostname":"buskin","pid":1590,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999907128s","next_cleanup":"Instant { tv_sec: 1587, tv_nsec: 297359212 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.000610728Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.000617888Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.000666889Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.00067767Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.0040452Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.004172713Z","hostname":"buskin","pid":1590,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV1 {\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV1 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.1.100,\n },\n ],\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.1.174,\n prefix: 24,\n },\n ),\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.004469369Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:979"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.004475239Z","hostname":"buskin","pid":1590,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.00453037Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.004578251Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.004599842Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.004608212Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1039"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.004636262Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.004648423Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.004664893Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.004672933Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1080"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.004676693Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.004684293Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.004702493Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.004730044Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.005452829Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.005591462Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.005598892Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1113"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.005634533Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.005643353Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.005787726Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.005940349Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.006011131Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 83429885-0785-47bc-b249-4b860f5e232d, rack_id: 2f1c0d8e-5768-4d21-9912-5823b0779f9c, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.006134223Z","hostname":"buskin","pid":1590,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:35069","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 83429885-0785-47bc-b249-4b860f5e232d, rack_id: 2f1c0d8e-5768-4d21-9912-5823b0779f9c, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.006251976Z","hostname":"buskin","pid":1590,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.006259226Z","hostname":"buskin","pid":1590,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.006296157Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.006301057Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:530"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.006304687Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/server.rs:44"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.006308557Z","hostname":"buskin","pid":1590,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"No ledger in /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.006317257Z","hostname":"buskin","pid":1590,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.006367988Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"No ledger in /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.006384959Z","hostname":"buskin","pid":1590,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.006390949Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:519"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.006395149Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:547"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.00884504Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3849"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.019846339Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.030243766Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.040624652Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.050330925Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.059302852Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.067633965Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.076571552Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.085183221Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.093671128Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.093727649Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:558"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.093732999Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.09373792Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:370"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.09374556Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.09375106Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:449","result":"DisksManagementResult { status: [] }"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.11439916Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.131643279Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.145907527Z","hostname":"buskin","pid":1590,"sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:352"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.145969728Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.145981209Z","hostname":"buskin","pid":1590,"sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:364"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.145993929Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 97.319322ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.1493967Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4e241531-ca60-4ca6-befe-bff8a2b7e94c\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:08:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4e241531-ca60-4ca6-befe-bff8a2b7e94c\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.15949616Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.183348187Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.183399038Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.18826454Z","hostname":"buskin","pid":1590,"sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:379"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.1882799Z","hostname":"buskin","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.18829121Z","hostname":"buskin","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.247874622Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 164.017001ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.248372723Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e5be9b84-a499-45dd-af08-61d71fdabd03\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:08:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e5be9b84-a499-45dd-af08-61d71fdabd03\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.258567555Z","hostname":"buskin","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.413672948Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 118.44773ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.414332712Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3eb5970f-7465-4425-881b-c266b157e876\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:08:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3eb5970f-7465-4425-881b-c266b157e876\" }"} {"msg":"Mounting oxi_57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.478084461Z","hostname":"buskin","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.490562661Z","hostname":"buskin","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.490638103Z","hostname":"buskin","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.490661333Z","hostname":"buskin","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.490686124Z","hostname":"buskin","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.533788132Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 98.909233ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.534397185Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3f06b418-eb4a-4c5c-8511-58e2ae639448\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:08:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3f06b418-eb4a-4c5c-8511-58e2ae639448\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.63443858Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 304.242298ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.63493483Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"173a28b5-f6bc-4154-8ed0-abdfe214e91c\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:08:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"173a28b5-f6bc-4154-8ed0-abdfe214e91c\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.940061481Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 164.177287ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:05.943024822Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3d42c008-6f09-486f-b9d6-fd3fb29c22df\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:08:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3d42c008-6f09-486f-b9d6-fd3fb29c22df\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.107239425Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 243.066375ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.10796437Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b8a32a64-2d9d-4f50-be21-ba0c40707f5d\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:08:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b8a32a64-2d9d-4f50-be21-ba0c40707f5d\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.205401751Z","hostname":"buskin","pid":1590,"sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.206044425Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:48878","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.206080265Z","hostname":"buskin","pid":1590,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.206169157Z","hostname":"buskin","pid":1590,"file":"oximeter/producer/src/lib.rs:284","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"oximeter/producer/src/lib.rs:284","interval":"30s","address":"[fd00:1122:3344:101::1]:48878","producer_id":"83429885-0785-47bc-b249-4b860f5e232d","route":"//83429885-0785-47bc-b249-4b860f5e232d"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:06.207530856Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"305.225635ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.253375751Z","hostname":"buskin","pid":1590,"component":"VmmReservoirManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.253417682Z","hostname":"buskin","pid":1590,"component":"VmmReservoirManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.253531695Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:993","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.253539365Z","hostname":"buskin","pid":1590,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.254312491Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.254345652Z","hostname":"buskin","pid":1590,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.254378622Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.254394013Z","hostname":"buskin","pid":1590,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.254637518Z","hostname":"buskin","pid":1590,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.254923253Z","hostname":"buskin","pid":1590,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.254958274Z","hostname":"buskin","pid":1590,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.1.100 }], addresses: [V4(Ipv4Network { addr: 10.151.1.174, prefix: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.254999525Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.255005365Z","hostname":"buskin","pid":1590,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:35069","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.431022314Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.431136837Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.43129148Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.431341741Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.431366611Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.431521045Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.431535065Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.431609396Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:06.43658246Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.47016605Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:37690"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.470317503Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.470403555Z","hostname":"buskin","pid":1590,"uri":"/inventory","method":"GET","req_id":"9afdab8a-c267-47d3-8b98-a9a1667fe469","remote_addr":"[fd00:1122:3344:101::1]:37690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":154,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.499364089Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:38153"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.4994399Z","hostname":"buskin","pid":1590,"uri":"/sled-role","method":"GET","req_id":"b36b9f41-6037-4cb9-8fbf-dcc93375ceba","remote_addr":"[fd00:1122:3344:101::1]:38153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":33,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.500968042Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.501019193Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.501406971Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.501621516Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:815"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:06.51525161Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"376.497263ms"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: 7ef7feac-0f9f-4a17-82c5-46b9880ca79b, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, pool_id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool) }, OmicronPhysicalDiskConfig { id: 86c3203b-68ad-45af-b5c5-0e0843da267f, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, pool_id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool) }, OmicronPhysicalDiskConfig { id: 1f752822-1a8d-468d-9e82-45da1528845f, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, pool_id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool) }, OmicronPhysicalDiskConfig { id: bf649aef-2770-454b-8a2e-06eed9835614, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, pool_id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool) }, OmicronPhysicalDiskConfig { id: 905c680c-1ae6-4335-bdbf-3b8f05b36f87, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, pool_id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool) }, OmicronPhysicalDiskConfig { id: 8bc4fd38-ee78-4f5f-8da2-b3df68bc1e54, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, pool_id: 7673312c-f216-4503-9437-30203b65ec1d (zpool) }, OmicronPhysicalDiskConfig { id: 5e0bfc9c-f342-4a48-8107-850cbbb58a60, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, pool_id: 4237ec86-747e-4b2e-a81d-30673b143f9c (zpool) }, OmicronPhysicalDiskConfig { id: 8c74ed5c-c937-4814-a831-67be9c5f0626, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, pool_id: 0a8cb798-6ba7-43a1-aa2f-7aba85873942 (zpool) }, OmicronPhysicalDiskConfig { id: 2ec6650d-7204-4efc-b36d-a3ed3edf35d5, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, pool_id: e211498b-7438-4a66-89c4-ebf7a721e12d (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.532474729Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:346"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.53253233Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:63809"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, id: 7ef7feac-0f9f-4a17-82c5-46b9880ca79b, pool_id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, id: 86c3203b-68ad-45af-b5c5-0e0843da267f, pool_id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, id: 1f752822-1a8d-468d-9e82-45da1528845f, pool_id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, id: bf649aef-2770-454b-8a2e-06eed9835614, pool_id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, id: 905c680c-1ae6-4335-bdbf-3b8f05b36f87, pool_id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, id: 8bc4fd38-ee78-4f5f-8da2-b3df68bc1e54, pool_id: 7673312c-f216-4503-9437-30203b65ec1d (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, id: 5e0bfc9c-f342-4a48-8107-850cbbb58a60, pool_id: 4237ec86-747e-4b2e-a81d-30673b143f9c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, id: 8c74ed5c-c937-4814-a831-67be9c5f0626, pool_id: 0a8cb798-6ba7-43a1-aa2f-7aba85873942 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, id: 2ec6650d-7204-4efc-b36d-a3ed3edf35d5, pool_id: e211498b-7438-4a66-89c4-ebf7a721e12d (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.532764555Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.532773105Z","hostname":"buskin","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.532790625Z","hostname":"buskin","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.532798366Z","hostname":"buskin","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:681"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.532807686Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.532815006Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.532823266Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.532832366Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.625780344Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:06.625814935Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"4ab38590-bc9f-4317-9a2d-902570f44129 (zpool)"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:06.989357782Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.089743688s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T05:08:07.419058489Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:07.610228123Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:07.781719008Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:07.78182886Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:07.78183818Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:07.781845811Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:07.781853081Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:07.782004644Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:07.782011614Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:08.151174969Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.811984711s"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:08.380859136Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:08.383788477Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T05:08:08.585261217Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Automatically destroying dataset: oxp_4ab38590-bc9f-4317-9a2d-902570f44129/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:08.590876964Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:08.755834382Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 67.607417ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:09.53053689Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:10.073289423Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"3.177305769s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T05:08:10.073353044Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 110.971714ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:10.073512078Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:10.501575951Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 205.22479ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:10.699278232Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 451.445895ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:11.309808637Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T05:08:11.311494433Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_4ab38590-bc9f-4317-9a2d-902570f44129","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:11.549226138Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 910.678462ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:11.988828061Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:11.989061086Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:11.989075676Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:11.989142877Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:11.989150988Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:11.989155408Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:12.216120968Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:12.216148999Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool)"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T05:08:12.440957825Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:12.440988356Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 915.632692ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:13.350556235Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:13.352404393Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"4.158246626s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T05:08:13.58444021Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:13.595805577Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:13.829881716Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:13.829935557Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:13.829944837Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:13.829960668Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:13.829976468Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:13.83010147Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:13.830111951Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:15.041407899Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_f2ad007c-d0d5-499a-a92a-f0e5d6ac476c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:15.545171159Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:15.772825625Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T05:08:15.772873016Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.524540393Z","hostname":"buskin","pid":1590,"component":"VmmReservoirManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"Looking for unencrypted datasets in oxp_f2ad007c-d0d5-499a-a92a-f0e5d6ac476c","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.636325203Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.644116926Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.644156137Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.644165937Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.644173207Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.644177867Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.653469861Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.653506992Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"eeebb608-8f09-47dd-a873-6005f1a33225 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.74360131Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.751416813Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.751465263Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.751471734Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.751512094Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.751522065Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.751636407Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.751642757Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T05:08:16.774950443Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.789763272Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_eeebb608-8f09-47dd-a873-6005f1a33225/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.798994824Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_eeebb608-8f09-47dd-a873-6005f1a33225","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.923582561Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.931435965Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.931465346Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.931488596Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.931494726Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.931498726Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.941205158Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:16.941223859Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.030867487Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.038032287Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.038066437Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.038072887Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.038107438Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.038114498Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.038343203Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.038356113Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:17.043375808Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.077984719Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_4917eed2-7e51-43de-bbf3-90e6e217a77b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.086084358Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_4917eed2-7e51-43de-bbf3-90e6e217a77b","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.212741498Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.221121313Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.221167644Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.221176094Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.221185814Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.221191624Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.232587942Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.232628563Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"137f7f84-2894-49f9-be7d-163348c068b9 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.329862439Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.337988329Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.3380421Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.33805095Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.338101781Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.338116562Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.338275475Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.338284475Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.374300016Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_137f7f84-2894-49f9-be7d-163348c068b9/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.38218795Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_137f7f84-2894-49f9-be7d-163348c068b9","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.505695925Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:17.512366534Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"19.40907577s"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.513537898Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.513552019Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.513568069Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.513574339Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.513580239Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.52321142Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.523241301Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"7673312c-f216-4503-9437-30203b65ec1d (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.622646642Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 7673312c-f216-4503-9437-30203b65ec1d (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.631248792Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.631296653Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.631310803Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.631325253Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.631337504Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.631446566Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.631462776Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.668219732Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_7673312c-f216-4503-9437-30203b65ec1d/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.676429854Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T05:08:17.780694217Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_7673312c-f216-4503-9437-30203b65ec1d","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.807018616Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.815556194Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.815619245Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.815640885Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.815649216Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.815656306Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.825161284Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.825194564Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"4237ec86-747e-4b2e-a81d-30673b143f9c (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.925856613Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 4237ec86-747e-4b2e-a81d-30673b143f9c (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.934119745Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.934176386Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.934183036Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.934203987Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.934214597Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.93436427Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.9343728Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.970667036Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_4237ec86-747e-4b2e-a81d-30673b143f9c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:17.979241135Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_4237ec86-747e-4b2e-a81d-30673b143f9c","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.104576277Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.113080305Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.113131346Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.113139766Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.113169866Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.113178547Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.123951151Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.123980212Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"0a8cb798-6ba7-43a1-aa2f-7aba85873942 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.219222737Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 0a8cb798-6ba7-43a1-aa2f-7aba85873942 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.227440959Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.22750499Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.22751479Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.227564161Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.227631503Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.227776906Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.227786776Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.267538054Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_0a8cb798-6ba7-43a1-aa2f-7aba85873942/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.276114403Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.387981005Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2303"} {"msg":"Looking for unencrypted datasets in oxp_0a8cb798-6ba7-43a1-aa2f-7aba85873942","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.409201557Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.418636004Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.418684585Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.418693185Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.418701715Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.418706225Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.429795596Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.429831917Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"e211498b-7438-4a66-89c4-ebf7a721e12d (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.531256811Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: e211498b-7438-4a66-89c4-ebf7a721e12d (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.5398237Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.539861671Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.539884501Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.539890891Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.539902332Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.540031404Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.540051075Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.575256849Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_e211498b-7438-4a66-89c4-ebf7a721e12d/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.583324227Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_e211498b-7438-4a66-89c4-ebf7a721e12d","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.706365891Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.714420699Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.71446907Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Writing ledger to /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.714512351Z","hostname":"buskin","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.714684285Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 07066414-66e9-4cbf-a342-0d2cd3c1d2ca (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: 7673312c-f216-4503-9437-30203b65ec1d (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: 4237ec86-747e-4b2e-a81d-30673b143f9c (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: 0a8cb798-6ba7-43a1-aa2f-7aba85873942 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: e211498b-7438-4a66-89c4-ebf7a721e12d (zpool), kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Writing ledger to /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.714842738Z","hostname":"buskin","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.715168785Z","hostname":"buskin","pid":1590,"uri":"/omicron-physical-disks","method":"PUT","req_id":"b4774c02-2b0a-4525-940c-338063cb8083","remote_addr":"[fd00:1122:3344:101::1]:63809","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":12182578,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.746679172Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.746857285Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:33180"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.747159942Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.747169352Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T05:08:18.781925836Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.799230617Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.799276908Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.849769691Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.849814812Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.900577659Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.900627321Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.900648521Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.900709272Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.94666631Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.946719961Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.946730442Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_bff3110b-f67b-4955-a822-d477134772d8","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:18.998242335Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_37798025-9c23-405f-8532-61a017646e4c","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:19.001802159Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:19.006012557Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_bff3110b-f67b-4955-a822-d477134772d8","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:19.019172921Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_37798025-9c23-405f-8532-61a017646e4c","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:19.024173775Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:19.028292091Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:19.043219483Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T05:08:19.784567865Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Profile for oxz_internal_dns_bff3110b-f67b-4955-a822-d477134772d8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:21.239599924Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:21.239716076Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:21.23991884Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_bff3110b-f67b-4955-a822-d477134772d8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_internal_dns_37798025-9c23-405f-8532-61a017646e4c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:21.730361243Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:22.211907551Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Profile for oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:22.212011703Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:22.212041323Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_bff3110b-f67b-4955-a822-d477134772d8\") failed: Property not found. retry in 46.666141ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:22.220102142Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_bff3110b-f67b-4955-a822-d477134772d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:22.220147712Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_37798025-9c23-405f-8532-61a017646e4c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:22.224785159Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_bff3110b-f67b-4955-a822-d477134772d8\") failed: Property not found. retry in 126.989745ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:22.277454107Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_bff3110b-f67b-4955-a822-d477134772d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165\") failed: Property not found. retry in 47.152487ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:22.361963908Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_37798025-9c23-405f-8532-61a017646e4c\") failed: Property not found. retry in 70.042389ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:22.368701119Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_37798025-9c23-405f-8532-61a017646e4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bff3110b-f67b-4955-a822-d477134772d8\") failed: Property not found. retry in 227.085964ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:22.413771988Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_bff3110b-f67b-4955-a822-d477134772d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165\") failed: Property not found. retry in 112.28862ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:22.422291026Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_37798025-9c23-405f-8532-61a017646e4c\") failed: Property not found. retry in 122.858212ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:22.540818467Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_37798025-9c23-405f-8532-61a017646e4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165\") failed: Property not found. retry in 151.010003ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:22.556845421Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bff3110b-f67b-4955-a822-d477134772d8\") failed: Property not found. retry in 301.779236ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:22.674215767Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_bff3110b-f67b-4955-a822-d477134772d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_37798025-9c23-405f-8532-61a017646e4c\") failed: Property not found. retry in 106.836722ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:22.691797594Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_37798025-9c23-405f-8532-61a017646e4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165\") failed: Property not found. retry in 207.378912ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:22.730747346Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_37798025-9c23-405f-8532-61a017646e4c\") failed: Property not found. retry in 473.658366ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:22.814117183Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_37798025-9c23-405f-8532-61a017646e4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165\") failed: Property not found. retry in 765.373988ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:22.947050714Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bff3110b-f67b-4955-a822-d477134772d8\") failed: Property not found. retry in 723.965452ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:22.987205361Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_bff3110b-f67b-4955-a822-d477134772d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_37798025-9c23-405f-8532-61a017646e4c\") failed: Property not found. retry in 923.390184ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:23.29756342Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_37798025-9c23-405f-8532-61a017646e4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bff3110b-f67b-4955-a822-d477134772d8\") failed: Property not found. retry in 510.59314ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:23.721069358Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_bff3110b-f67b-4955-a822-d477134772d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165\") failed: Property not found. retry in 505.521673ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:23.72883423Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_37798025-9c23-405f-8532-61a017646e4c\") failed: Property not found. retry in 585.411716ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:24.230600469Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_37798025-9c23-405f-8532-61a017646e4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bff3110b-f67b-4955-a822-d477134772d8\") failed: Property not found. retry in 1.346857427s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:24.240893443Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_bff3110b-f67b-4955-a822-d477134772d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165\") failed: Property not found. retry in 807.35428ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:24.248716566Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:9fb2::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:24.440948973Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2320"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:24.440976474Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"wait for service Some(\"oxz_internal_dns_37798025-9c23-405f-8532-61a017646e4c\") failed: Property not found. retry in 879.195191ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:24.826753615Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_37798025-9c23-405f-8532-61a017646e4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe52:4b01","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:24.996257957Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2327"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:25.002843005Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2388"} {"msg":"wait for service Some(\"oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165\") failed: Property not found. retry in 1.48965367s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:25.065356518Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:25.134478469Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2573"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:25.393230072Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2737"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:25.582526168Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2452"} {"msg":"wait for service Some(\"oxz_internal_dns_bff3110b-f67b-4955-a822-d477134772d8\") failed: Property not found. retry in 1.491677663s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:25.597820377Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_bff3110b-f67b-4955-a822-d477134772d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_37798025-9c23-405f-8532-61a017646e4c\") failed: Property not found. retry in 1.202131163s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:25.715089661Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_37798025-9c23-405f-8532-61a017646e4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:25.879949937Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2487"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:26.074298239Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2787"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:26.248557911Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2814"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:26.458867544Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2681"} {"msg":"wait for service Some(\"oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165\") failed: Property not found. retry in 1.127822804s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:26.563872643Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:26.630433261Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2481"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:26.640572572Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3873","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:26.640622133Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3889"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:26.640632253Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"wait for service Some(\"oxz_internal_dns_37798025-9c23-405f-8532-61a017646e4c\") failed: Property not found. retry in 1.41801255s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:26.926997673Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_37798025-9c23-405f-8532-61a017646e4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bff3110b-f67b-4955-a822-d477134772d8\") failed: Property not found. retry in 853.487714ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:27.099234132Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_bff3110b-f67b-4955-a822-d477134772d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.192842863Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3897"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.199100124Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3953"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.34332218Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:4015"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.443971558Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3996","rack_subnet":"fd00:1122:3344::"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.467058279Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:4043"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.57359462Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:4070"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.699037855Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:364"} {"msg":"wait for service Some(\"oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165\") failed: Property not found. retry in 868.804364ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:27.699820211Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.731385269Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:412"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.763625831Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:589"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.763716123Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:627"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.764163852Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:630","uptime":"2"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.764174632Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:434","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.842507325Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.842576837Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.842584877Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.842591807Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.842597557Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.842603247Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.842607637Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.842622118Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.842627768Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.842633058Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.842638458Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.842642888Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.842650718Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.842656328Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.842661728Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.842668328Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.842673719Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.842678959Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.842683369Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.842697059Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.842701549Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.842705759Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:27.842709949Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:28.591253472Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_bff3110b-f67b-4955-a822-d477134772d8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:28.591315633Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_37798025-9c23-405f-8532-61a017646e4c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:28.591322633Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_436d117c-1c16-43a3-b432-02e432890165"} {"msg":"Writing ledger to /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:28.591328574Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:28.59161715Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:28.591879975Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"186fdd70-80bf-4977-b411-2757e1ac88c9","remote_addr":"[fd00:1122:3344:101::1]:33180","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9844935,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:28.593942978Z","hostname":"buskin","pid":1590,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:547"} {"msg":"failed to write DNS configuration (will retry in 95.291753ms)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:28.982244512Z","hostname":"buskin","pid":1590,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:567","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.133209138Z","hostname":"buskin","pid":1590,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:547"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.198054Z","hostname":"buskin","pid":1590,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:547"} {"msg":"failed to write DNS configuration (will retry in 79.621957ms)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:29.236235166Z","hostname":"buskin","pid":1590,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:567","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:3::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.352278705Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:583"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.352295365Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.384405364Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.385396115Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.385402185Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.413784867Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.413920719Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:51390"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.415206696Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.415212737Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.436807487Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.436876508Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.458848696Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.458909067Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.478285561Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.478389793Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.478614618Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.478775811Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.491973566Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.524117567Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.525024285Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.525034616Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.569544343Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 216, 228]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.569593755Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.570173936Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.570187007Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.570522014Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.571729429Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.595240669Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:29.614396478Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:30.781457405Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:30.781582737Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55\") failed: Property not found. retry in 38.653552ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:30.928279805Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55\") failed: Property not found. retry in 120.221844ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:30.975697423Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55\") failed: Property not found. retry in 204.904323ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:31.125398604Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:31.181773799Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60317"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:31.181937542Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"cdc5dbd6-8033-428b-8920-3d05b3706c41","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60317","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":79,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55\") failed: Property not found. retry in 241.144979ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:31.340951527Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55\") failed: Property not found. retry in 1.175926024s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:31.591881517Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55\") failed: Property not found. retry in 781.233356ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:32.777632453Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55\") failed: Property not found. retry in 1.174224976s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:33.568574349Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55\") failed: Property not found. retry in 1.223039617s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:34.752506717Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55\") failed: Property not found. retry in 1.003677295s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:35.984263432Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:37.009558283Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55"} {"msg":"Writing ledger to /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:37.009621284Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:37.010001562Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:37.010343239Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"f2c4cf04-6539-4c12-9a6b-767fdc37ad8c","remote_addr":"[fd00:1122:3344:101::1]:51390","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7596351,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:37.012449373Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:624"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:37.040929627Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:37.04108649Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:52711"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:37.051262672Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:37.051388895Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"8749799f-7032-4807-8a71-96cbf5575906","remote_addr":"[fd00:1122:3344:101::1]:52711","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":10221,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:37.053935158Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:37.196226274Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:37.196349816Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:35140"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:41.207184398Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:41.207305741Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"5831184c-9d6e-4444-aeca-f47dd98d330a","remote_addr":"[fd00:1122:3344:101::1]:35140","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4010858,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:41.210216341Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:41.386152099Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:41.386289091Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:42046"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:41.39483836Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:41.395022224Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"5df89745-2981-49d1-81ae-b58cb10e7151","remote_addr":"[fd00:1122:3344:101::1]:42046","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8625,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:41.398569918Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:41.638815855Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:41.638929828Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:46917"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_43617b58-d1ff-4c18-a5ff-af7f04834d55': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:41.648044698Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:41.648208161Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"8130da3d-1c2c-4b49-bf99-99d4b198b123","remote_addr":"[fd00:1122:3344:101::1]:46917","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9157,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:41.650827306Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:41.776609427Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:41.77671214Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:42564"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:43.785870619Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"0bfab808-189b-4cdc-ac47-53f2b77aa7d5","remote_addr":"[fd00:1122:3344:101::1]:42564","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":2009004,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714540122.466462, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:43.788288139Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:43.788295269Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:43.98455026Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:43.984664162Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:45864"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:43.993054137Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"e3eb936d-b57a-4b0a-8705-b212c59cc957","remote_addr":"[fd00:1122:3344:101::1]:45864","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8292,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714540122.466462, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:43.994786853Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:43.994793363Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:44.156396652Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:44.156500364Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:46313"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:44.164543212Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"339dbafa-ba37-49ac-97b3-eaf72df2de96","remote_addr":"[fd00:1122:3344:101::1]:46313","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7929,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714540122.466462, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:44.166319518Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:44.166324959Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:44.401151624Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:44.401252956Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:49159"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:44.409307714Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"d977fd92-71b3-4060-8f6c-147d06bdd877","remote_addr":"[fd00:1122:3344:101::1]:49159","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7906,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714540122.466462, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:44.410956918Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:44.410964528Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:44.800596Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:44.800756823Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:44317"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:44.810427955Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"e43c26d8-5d10-439c-857d-a95ca1472b73","remote_addr":"[fd00:1122:3344:101::1]:44317","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9609,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714540122.466462, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:44.812233062Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:44.812238662Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:45.087219764Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:45.087343726Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:52137"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:45.095308472Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"4fbfc144-5e06-4f3a-a187-0aae98403bcd","remote_addr":"[fd00:1122:3344:101::1]:52137","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7864,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714540122.466462, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:45.098778585Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:45.098788555Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:45.463840494Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:45.463940576Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:60184"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:45.471900482Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"2adf2b3f-8985-4e79-95d1-2207b6defacd","remote_addr":"[fd00:1122:3344:101::1]:60184","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7815,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714540122.466462, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:45.47417427Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:45.47418503Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:45.990834879Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:45.99092805Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:48995"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:46.00004644Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"2d68f1e1-4f82-4716-80e6-3fc85861728f","remote_addr":"[fd00:1122:3344:101::1]:48995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9018,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714540122.466462, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:46.00196592Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:46.001973031Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:46.492741001Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:46.492833762Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:45181"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:46.501744298Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"7771782d-cad3-4049-9b2f-47a649b6c344","remote_addr":"[fd00:1122:3344:101::1]:45181","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8785,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714540122.466462, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:46.504769731Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:46.504780221Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:47.651759039Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:47.651862421Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:37729"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:47.660172904Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"eebbe984-c5f0-4625-b91a-f9bcebb8ff06","remote_addr":"[fd00:1122:3344:101::1]:37729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8211,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714540122.466462, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:47.662118715Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:47.662124265Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:48.699164744Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:48.699279766Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:41534"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:48.702628536Z","hostname":"buskin","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:48.72204895Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"f178ecf7-caa9-4629-8cc2-6a43d6905472","remote_addr":"[fd00:1122:3344:101::1]:41534","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":22582,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 1241950280, ip_addr: 74.6.168.72, stratum: 3, ref_time: 1714540127.1596067, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:48.723690355Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:48.723710535Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1192"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:48.752552016Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:48.752680359Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:62574"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:48.752945125Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:48.752962025Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:48.773065364Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:48.773118605Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:48.790977717Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:48.791033468Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:48.81173493Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:48.811778181Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:48.884720201Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:48.884765002Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:48.950998143Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:48.951053744Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.001388008Z","hostname":"buskin","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.010368377Z","hostname":"buskin","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.033040289Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.03308053Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.099404942Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.099456774Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.164329356Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.164398467Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.164671903Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.164835876Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:49.198070459Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:49.19811468Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:49.19812367Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:49.19813022Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:49.198138691Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.198289314Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"d550faa6-e491-4550-96e2-887e563f05dd","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized)]","latency_us":431531,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:49.198523628Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"98.523861ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d550faa6-e491-4550-96e2-887e563f05dd\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:08:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d550faa6-e491-4550-96e2-887e563f05dd\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.298667766Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.299118015Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.299133726Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.317471568Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.317522819Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.335978454Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.336021845Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.354185863Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.354236384Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.372797401Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.372835422Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.391391109Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.39143414Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.409757902Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.409805043Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.428065543Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.428109734Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.446350234Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.446414306Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.44661989Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.446773563Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:49.478676838Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:49.478735959Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:49.478746069Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:49.47875771Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:49.47876883Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.478862332Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"3eb266a7-e682-4ba7-b70d-9c22c3e75ec7","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized), (\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized), (\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized)]","latency_us":179931,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:49.479052726Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"166.611647ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3eb266a7-e682-4ba7-b70d-9c22c3e75ec7\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:08:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3eb266a7-e682-4ba7-b70d-9c22c3e75ec7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.646404894Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.646751911Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.646777542Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.666810519Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.66685767Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.684909186Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.684955187Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.703086795Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.703131276Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.720667792Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.720706063Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.737490982Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.737530853Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.756127631Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.756169362Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.775332851Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.775378932Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.793596072Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.793652363Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.793832757Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.793950699Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:49.825449706Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:49.825508317Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:49.825518377Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:49.825527797Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:49.825537077Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.82564982Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"96e15f4f-d8a1-438f-9229-14305e14c7b3","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized), (\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized)]","latency_us":179080,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:49.825854964Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"166.277604ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"96e15f4f-d8a1-438f-9229-14305e14c7b3\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:08:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"96e15f4f-d8a1-438f-9229-14305e14c7b3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.993220453Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.99356684Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:49.993584811Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.01178431Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.011823431Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.032289559Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.03234099Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.049490328Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.0495424Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.067843387Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.067891963Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.085520052Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.085562612Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.104381386Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.104428997Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.122537365Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.122579676Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.139262575Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.139310666Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.13953116Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.139650717Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:50.171920477Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:50.171970958Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:50.171979419Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:50.171986319Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:50.171995129Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.172108631Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"c70b130b-6534-4efc-bf1c-d85e0da676c9","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized), (\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized)]","latency_us":179173,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:50.172302955Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"210.17892ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c70b130b-6534-4efc-bf1c-d85e0da676c9\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:08:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c70b130b-6534-4efc-bf1c-d85e0da676c9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.382878656Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.383214783Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.383249474Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.401874964Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.401914924Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.422208109Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.422248949Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.441526092Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.441570003Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.460410547Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.460464628Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.478848762Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.478892423Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.49739043Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.497429511Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.515513889Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.515553509Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.532513704Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.532568734Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.532759019Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.532892752Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:50.563597784Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:50.563644185Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:50.563671525Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:50.563681686Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:50.563693336Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.563777997Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"77d41fee-1928-4d95-b4ae-89dc33f11ff4","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized), (\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized)]","latency_us":181211,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:50.563985512Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"224.244967ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"77d41fee-1928-4d95-b4ae-89dc33f11ff4\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:08:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"77d41fee-1928-4d95-b4ae-89dc33f11ff4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.788771679Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.790048516Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.790060326Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.808462411Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.808502802Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.827229363Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.827281515Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.845694529Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.845739887Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.864046313Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.864081453Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.882671572Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.882713933Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.901089627Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.901125588Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.919729467Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.919767788Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.938482441Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.93852056Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.938734544Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.938875697Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:50.971285084Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:50.971352976Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:50.971365326Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:50.971372636Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:50.971380256Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:50.971449438Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"5a2e24dd-7358-4fec-b207-a1348ad90ab0","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized), (\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized)]","latency_us":183028,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:50.971631412Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"152.438101ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5a2e24dd-7358-4fec-b207-a1348ad90ab0\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:08:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5a2e24dd-7358-4fec-b207-a1348ad90ab0\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.124342493Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.124638179Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.12465351Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.143538894Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.143579175Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.162384698Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.162440449Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.181741383Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.181789384Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.200559766Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.200602747Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.21937791Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.21941546Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.237652912Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.237695122Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.255841542Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.255878433Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.274192555Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.274248346Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.2744094Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.274509512Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:51.30696942Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:51.307018121Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:51.307027771Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:51.307037012Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:51.307048312Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.307117893Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"f9bb008f-0ebc-47e4-95a3-d32c4d545977","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized), (\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized)]","latency_us":183141,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:51.307297827Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"172.594457ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f9bb008f-0ebc-47e4-95a3-d32c4d545977\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:08:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f9bb008f-0ebc-47e4-95a3-d32c4d545977\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.480035267Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.480333714Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.480409715Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.499404632Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.499452883Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.517819267Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.517857118Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.536320104Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.536364525Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.555000514Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.555035155Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:51.573384449Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"288.554601ms"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.57391845Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.57394301Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.592206982Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.592242612Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.61125302Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.61129493Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.629990391Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.630031282Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.630243066Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.630381429Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:51.663654735Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:51.663709596Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:51.663719666Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:51.663739357Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:51.663749077Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.663848319Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"456edc91-1d4d-46e8-a943-b657dc7a4fd1","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized), (\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized)]","latency_us":184165,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:51.664055563Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"268.41514ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"456edc91-1d4d-46e8-a943-b657dc7a4fd1\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:08:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"456edc91-1d4d-46e8-a943-b657dc7a4fd1\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.933040195Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.933178868Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.933191738Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.952105034Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.952149845Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.970468268Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.970525649Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.989036016Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:51.989079386Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.007382119Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.00742003Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.026351196Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.026393996Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.04476432Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.044804571Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.063344379Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.06340163Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.081962708Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.082021449Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.082177122Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.082287985Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:52.115206283Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:52.115258464Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:52.115269984Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:52.115276054Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:52.115283444Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.115350186Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"88ce4f67-8b9e-4eb3-bd44-3e6e2c1f5c62","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized)]","latency_us":182762,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:52.115506259Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"563.44627ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"88ce4f67-8b9e-4eb3-bd44-3e6e2c1f5c62\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:08:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"88ce4f67-8b9e-4eb3-bd44-3e6e2c1f5c62\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.678198729Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.678520046Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.678540967Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.699396503Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.699442893Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.717849378Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.717905849Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.736860326Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.736904947Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.757378314Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.757434565Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.776734109Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.77678155Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.795235966Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.795279627Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.814200692Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.814238973Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.833467445Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.833531556Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.833752761Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.833901734Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:52.866831772Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:52.866883563Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:52.866892203Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:52.866900333Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:52.866907173Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:52.867055167Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"72757577-111c-472e-933b-ac028865174c","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized)]","latency_us":189220,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:52.86721221Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"518.482409ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"72757577-111c-472e-933b-ac028865174c\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:08:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"72757577-111c-472e-933b-ac028865174c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:53.385182506Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:53.385437771Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:53.385448551Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:53.406297257Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:53.406361699Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:53.425392586Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:53.425438247Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:53.443765821Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:53.443813902Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:53.462595264Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:53.462636045Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:53.481571311Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:53.481614421Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:53.499791641Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:53.499853253Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:53.518642575Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:53.518685986Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:53.536125631Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:53.536187822Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:53.536352556Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:53.536464825Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:53.568766723Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:53.568834764Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:53.568844645Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:53.568852445Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:53.568859845Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:53.568928406Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"153da1e1-37a7-4383-9946-ba142227428b","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized), (\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized)]","latency_us":184127,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:53.56908438Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"555.800977ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"153da1e1-37a7-4383-9946-ba142227428b\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:08:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"153da1e1-37a7-4383-9946-ba142227428b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.123768662Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.123980877Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.123989447Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.144575117Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.144612468Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.164230318Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.164275819Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.18297334Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.183033901Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.202349895Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.202385556Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.221185378Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.221228329Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.239207855Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.239250146Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.256846864Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.256886495Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.274155165Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.274213297Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.274422651Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.274565594Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:54.306217016Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:54.306256996Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:54.306265237Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:54.306273237Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:54.306281057Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.30639711Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"4ba7f8de-f7b0-4a23-acb4-3ece0c5c0e56","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized), (\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized), (\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized)]","latency_us":183012,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:54.306543322Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"596.862458ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4ba7f8de-f7b0-4a23-acb4-3ece0c5c0e56\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:08:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4ba7f8de-f7b0-4a23-acb4-3ece0c5c0e56\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.903336886Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.903601301Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.903649842Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.923109909Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.92317587Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.941047704Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.941080115Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.958791295Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.958832746Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.977875213Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.977917594Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.996754898Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:54.996808579Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.016844228Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.016883329Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.036194352Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.036236953Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.054928674Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.054983795Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.055159309Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.055267681Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:55.087177058Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:55.087218339Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:55.087226639Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:55.087243119Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:55.087251457Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.087329711Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"b995c04a-589b-458b-9e39-46929d8f74c1","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized)]","latency_us":184372,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:55.087482034Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"639.074324ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b995c04a-589b-458b-9e39-46929d8f74c1\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:08:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b995c04a-589b-458b-9e39-46929d8f74c1\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.726186173Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.726418268Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.726435749Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.746245493Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.746284044Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.764561236Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.764615177Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.783075842Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.783119943Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.801925296Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.801967657Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.820215129Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.82024835Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.838545922Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.838582043Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.857742743Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.857793924Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.875572066Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.875630557Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.875821251Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.875966624Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:55.909301841Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:55.909362862Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:55.909372982Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:55.909380293Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:55.909389663Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:55.909492865Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"d927905e-c2db-476c-8a9f-ded6cc8a3bb1","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized), (\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized)]","latency_us":183706,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:55.909678419Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.146483168s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d927905e-c2db-476c-8a9f-ded6cc8a3bb1\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:08:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d927905e-c2db-476c-8a9f-ded6cc8a3bb1\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:57.054449975Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:57.054861813Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:57.054874974Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:57.076940085Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:57.076980086Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:57.096593135Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:57.096632516Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:57.115716995Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:57.115755206Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:57.134548889Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:57.13460693Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:57.153191789Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:57.153229609Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:57.171498401Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:57.171534902Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:57.189256692Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:57.189293403Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:57.207059764Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:57.207107295Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:57.20731299Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:57.207432702Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:57.239799819Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:57.239848Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:57.23985617Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:57.23986581Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:57.2398777Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:57.239972712Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"97ffa261-115f-43b3-ba7e-6e56db00fe5a","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized), (\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized)]","latency_us":185834,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:57.240151216Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.893552905s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"97ffa261-115f-43b3-ba7e-6e56db00fe5a\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:08:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"97ffa261-115f-43b3-ba7e-6e56db00fe5a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:59.129658818Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:59.129931483Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:59.129949153Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:59.150310709Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:59.15036318Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:59.170244436Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:59.170287047Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:59.189091769Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:59.189134801Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:59.208477775Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:59.208515555Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:59.22786519Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:59.227897461Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:59.246491289Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:59.24653956Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:59.266543888Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:59.266581759Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:59.285195478Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:59.285228809Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:59.285440893Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:59.285558326Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:59.318707579Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:59.31875947Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:59.31876973Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:59.31877963Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:59.31879132Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:08:59.318873022Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"a8817f03-0409-4676-830b-a6e3d5330f97","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized), (\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized)]","latency_us":189597,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:08:59.319051406Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"949.442002ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a8817f03-0409-4676-830b-a6e3d5330f97\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:08:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a8817f03-0409-4676-830b-a6e3d5330f97\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:00.267948478Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:00.268166603Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:00.268182513Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:00.28714741Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:00.287185887Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:00.305781549Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:00.3058235Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:00.324115489Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:00.324171264Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:00.342299602Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:00.342344383Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:00.360509393Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:00.360552674Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:00.379144142Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:00.379187393Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:00.396190149Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:00.39623342Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:00.415125894Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:00.415168945Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:00.41540399Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:00.415552323Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:00.44747209Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:00.447521502Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:00.447531592Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:00.447540522Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:00.447549852Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:00.447644974Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"89890f19-6445-484e-98aa-646899f1c2e4","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized), (\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized)]","latency_us":180088,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:00.447819628Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"2.697374751s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"89890f19-6445-484e-98aa-646899f1c2e4\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:09:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"89890f19-6445-484e-98aa-646899f1c2e4\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:00.803849109Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34589"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:00.80391623Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"1a110f67-58b9-4c18-b1da-102c5ba05b19","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34589","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:03.139207809Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:03.139383952Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:03.139397283Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:03.16127111Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:03.16130703Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:03.181323579Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:03.18135874Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:03.200329421Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:03.200366947Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:03.217994175Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:03.218033766Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:03.233598112Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:03.233630972Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:03.252383754Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:03.252410145Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:03.269832209Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:03.269864939Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:03.288708243Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:03.288748714Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:03.288974869Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:03.289091354Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:03.320867656Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:03.320918617Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:03.320927197Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:03.320935907Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:03.320944117Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:03.321025189Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"82ca8bfd-2756-4177-9780-6133ddd1f118","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized), (\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized)]","latency_us":182231,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:03.321210083Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.923517664s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"82ca8bfd-2756-4177-9780-6133ddd1f118\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:09:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"82ca8bfd-2756-4177-9780-6133ddd1f118\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:05.240630559Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:05.240980536Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:05.240999247Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:05.260961064Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:05.261009545Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:05.279950451Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:05.280000452Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:05.298149591Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:05.298187112Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:05.317936975Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:05.317977886Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:05.336713927Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:05.336759508Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:05.355386327Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:05.355432948Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:05.372013705Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:05.372056056Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:05.38947689Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:05.389531321Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:05.389727155Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:05.389861808Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:05.421751465Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:05.421801226Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:05.421811206Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:05.421820616Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:05.421827916Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:05.421970809Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"c022a9e6-0bb4-4e88-9a4a-42d5b8d167a3","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized), (\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized)]","latency_us":181675,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:05.422192544Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.872767812s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c022a9e6-0bb4-4e88-9a4a-42d5b8d167a3\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:09:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c022a9e6-0bb4-4e88-9a4a-42d5b8d167a3\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:06.855202894Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"745.702836ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:07.290219606Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:07.290506183Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:07.290518953Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:07.311104293Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:07.311151394Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:07.331703993Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:07.331750765Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:07.350935315Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:07.350984436Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:07.370580716Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:07.370626417Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:07.389700366Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:07.389745137Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:07.408296474Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:07.408339635Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:07.425881882Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:07.425939773Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:07.444261336Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:07.444319637Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:07.444478761Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:07.444589723Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:07.477489631Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:07.477541532Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:07.477551692Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:07.477561072Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:07.477566912Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:07.477658314Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"09b513f3-9e54-4e85-bc0e-21d209e7bab1","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized), (\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized)]","latency_us":187824,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:07.477880189Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"4.277022282s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"09b513f3-9e54-4e85-bc0e-21d209e7bab1\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:09:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"09b513f3-9e54-4e85-bc0e-21d209e7bab1\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:11.744710887Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:11.745050854Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:11.745062524Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:11.767169427Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:11.767200027Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:11.787187175Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:11.787225755Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:11.807006069Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:11.807038844Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:11.823950653Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:11.823988789Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:11.841199914Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:11.841234274Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:11.860612619Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:11.86066294Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:11.878512443Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:11.878553624Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:11.896490359Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:11.89653756Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:11.896746855Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:11.896868317Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:11.928923197Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:11.928999329Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:11.929009369Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:11.929017619Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:11.929025989Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:11.929102711Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"179b0a8f-ada9-4144-bf8d-3caf092d0281","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized), (\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized)]","latency_us":184738,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:11.929296225Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"4.983610167s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"179b0a8f-ada9-4144-bf8d-3caf092d0281\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:09:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"179b0a8f-ada9-4144-bf8d-3caf092d0281\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:16.900244069Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:16.900509285Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:16.900522815Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:16.921647857Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:16.921689038Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:16.942077024Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:16.942113834Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:16.962252875Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:16.962287346Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:16.980581438Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:16.980618979Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:16.998116875Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:16.998156266Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:17.014875895Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:17.014919846Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:17.031154715Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:17.031193176Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:17.046494846Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:17.046531867Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:17.046745251Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:17.046861162Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:17.07633358Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:17.076389301Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:17.076398701Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:17.076404591Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:17.076411651Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:17.076481683Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"663a6811-45e8-4af7-b2ed-286d6d96cd56","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized), (\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized)]","latency_us":176572,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:17.076665607Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"2.775751275s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"663a6811-45e8-4af7-b2ed-286d6d96cd56\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:09:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"663a6811-45e8-4af7-b2ed-286d6d96cd56\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:19.84568358Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:19.846003997Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:19.846097699Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:19.867422375Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:19.867472786Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:19.886785049Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:19.88683028Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:19.906554103Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:19.906597404Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:19.927036101Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:19.927076804Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:19.945765432Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:19.945808623Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:19.964558995Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:19.964595766Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:19.982751475Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:19.982788896Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:20.002764023Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:20.002828135Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:20.003004459Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:20.003130771Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:20.037129482Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:20.037183723Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:20.037191863Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:20.037201113Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:20.037207113Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:20.037299105Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"16d09f9d-9125-4529-8ae0-c02cebcafb26","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized), (\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized)]","latency_us":191998,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:20.037490709Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"6.765265577s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"16d09f9d-9125-4529-8ae0-c02cebcafb26\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:09:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"16d09f9d-9125-4529-8ae0-c02cebcafb26\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:22.594492921Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"1.35660254s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:26.785529518Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:26.785828312Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:26.785857692Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:26.808357583Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:26.808399384Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:26.82830677Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:26.828349531Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:26.847351875Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:26.847401809Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:26.866206562Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:26.866245063Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:26.885381473Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:26.885420844Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:26.9029507Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:26.902981911Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:26.921915096Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:26.921965807Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:26.940229859Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:26.94026935Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:26.940460884Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:26.940578786Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:26.972974463Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:26.973028337Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:26.973043637Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:26.973053175Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:26.973060355Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:26.973097036Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"4356ca91-a8f0-4823-8ff1-2f2313487393","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized), (\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized), (\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized)]","latency_us":187948,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:26.97326797Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"9.778983737s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4356ca91-a8f0-4823-8ff1-2f2313487393\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:09:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4356ca91-a8f0-4823-8ff1-2f2313487393\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:30.756530991Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46619"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:30.756636213Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"6583d133-e3f1-4b49-b76b-609e154c28cf","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46619","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":60,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:36.726757201Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:36.72718659Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:36.72720634Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:36.749312882Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:36.749362633Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:36.76931847Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:36.769358511Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:36.788861269Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:36.7889087Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:36.807025849Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:36.807061189Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:36.824197257Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:36.824246309Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:36.84347154Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:36.843510141Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:36.860862694Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:36.860898975Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:36.879246708Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:36.879305449Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:36.879501373Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:36.879632416Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:36.910472891Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:36.910529172Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:36.910537822Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:36.910544392Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:36.910560543Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:36.910700076Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"8868978e-f902-47ed-81aa-0e8129573e46","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized), (\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized)]","latency_us":184208,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:36.91092331Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"6.847551081s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8868978e-f902-47ed-81aa-0e8129573e46\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:09:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8868978e-f902-47ed-81aa-0e8129573e46\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:38.941315196Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"2.49631135s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:43.740535751Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:43.740838668Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:43.740851328Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:43.762825288Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:43.762873929Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:43.783438538Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:43.783489819Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:43.802803693Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:43.802841454Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:43.820700287Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:43.820741308Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:43.840697025Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:43.840735336Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:43.85909687Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:43.85914021Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:43.874814928Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:43.874853069Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:43.892737213Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:43.892782114Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:43.892988638Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:43.893136021Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:43.924784073Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:43.924846034Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:43.924855344Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:43.924862714Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:43.924870294Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:43.924938646Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"df89c9aa-7b5f-4cf1-863e-4e1880f16b65","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized)]","latency_us":184773,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:43.92515925Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"7.710333596s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"df89c9aa-7b5f-4cf1-863e-4e1880f16b65\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:09:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"df89c9aa-7b5f-4cf1-863e-4e1880f16b65\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:51.615611694Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:51.615865239Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:51.615881609Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:51.637798338Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:51.637841539Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:51.657476499Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:51.65752081Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:51.676303582Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:51.676350203Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:51.694532813Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:51.694575694Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:51.713656473Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:51.713706874Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:51.733433886Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:51.733476037Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:51.751593746Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:51.751625307Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:51.770244016Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:51.770286477Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:51.770494101Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:51.770610754Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:51.802585052Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:51.802628333Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:51.802635303Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:51.802643273Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:51.802651303Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:09:51.802719114Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"6524d324-887b-43e6-a8fb-05c8f9948c65","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized), (\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized)]","latency_us":187506,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:51.802892948Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"12.72584828s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6524d324-887b-43e6-a8fb-05c8f9948c65\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:09:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6524d324-887b-43e6-a8fb-05c8f9948c65\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:09:56.426858311Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"4.269912201s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:00.715753225Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51920"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:00.715829642Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"d3994fdd-3b2f-4c0b-a3b0-a1350109cd8e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51920","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":54,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:04.496145753Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:04.496391888Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:04.496408758Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:04.518574621Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:04.518611292Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:04.539347936Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:04.539379341Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:04.558503936Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:04.558540937Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:04.577738468Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:04.577782509Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:04.596818977Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:04.596852577Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:04.615056848Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:04.615094589Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:04.634050655Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:04.634088306Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:04.652157484Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:04.652213635Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:04.65247428Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:04.652618993Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:04.684349936Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:04.684394557Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:04.684403287Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:04.684411328Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:04.684417678Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:04.68452102Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"53136633-f396-46dc-9fb0-a88bf4162a0e","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized)]","latency_us":188775,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:04.684700834Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"10.460047322s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"53136633-f396-46dc-9fb0-a88bf4162a0e\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:10:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"53136633-f396-46dc-9fb0-a88bf4162a0e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:15.118002446Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:15.118376294Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:15.118388754Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:15.140916135Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:15.140972486Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:15.160446643Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:15.160503769Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:15.179507532Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:15.179560503Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:15.196802213Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:15.196846764Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:15.213289018Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:15.213329448Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:15.2287145Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:15.228763781Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:15.245229715Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:15.245273185Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:15.261442574Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:15.261494755Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:15.26172884Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:15.261849022Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:15.291198446Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:15.291253927Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:15.291265328Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:15.291272868Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:15.291278738Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:15.291355669Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"558f7ce0-ce0d-4cda-9de7-3c2022a21f28","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized), (\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized), (\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized)]","latency_us":173648,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:15.291550083Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"14.97537167s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"558f7ce0-ce0d-4cda-9de7-3c2022a21f28\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:10:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"558f7ce0-ce0d-4cda-9de7-3c2022a21f28\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:15.679549003Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"4.257309127s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:30.227996284Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:30.228170818Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:30.228182928Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:30.25076645Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:30.250824331Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:30.271440012Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:30.27148526Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:30.290877669Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:30.290926019Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:30.309009997Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:30.309051788Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:30.327367831Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:30.327413062Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:30.346138763Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:30.346174844Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:30.364415475Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:30.364458086Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:30.383030194Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:30.383074715Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:30.383354681Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:30.383496834Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:30.416123176Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:30.416179137Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:30.416189287Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:30.416198867Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:30.416208248Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:30.416294709Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"bf4b1717-28c0-4d9d-ab9d-90167e642ce1","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized), (\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized)]","latency_us":188746,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:30.416480153Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"24.066977386s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bf4b1717-28c0-4d9d-ab9d-90167e642ce1\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:10:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bf4b1717-28c0-4d9d-ab9d-90167e642ce1\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:30.667832367Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44293"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:30.667889818Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"4d2c6c87-ca18-4a0b-a643-ef0c8e2146dd","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44293","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":30,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:34.921254196Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"13.436070231s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:54.419386549Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:54.419752856Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:54.419765646Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:54.442365149Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:54.44240918Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:54.461769693Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:54.461828496Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:54.481489077Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:54.481529857Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:54.498713257Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:54.498757908Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:54.517908068Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:54.517954449Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:54.53476721Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:54.534811421Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:54.552337108Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:54.552381308Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:54.570578739Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:54.57063789Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:54.571004738Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:54.57113604Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:54.603287662Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:54.603336543Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:54.603345903Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:54.603354604Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:54.603363004Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:10:54.603489557Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"b28d3423-f99a-4c8f-9049-9b0398675c79","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\", TimeNotSynchronized), (\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\", TimeNotSynchronized), (\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\", TimeNotSynchronized), (\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\", TimeNotSynchronized), (\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\", TimeNotSynchronized)]","latency_us":184457,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:10:54.603694541Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"30.827528075s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b28d3423-f99a-4c8f-9049-9b0398675c79\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:10:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b28d3423-f99a-4c8f-9049-9b0398675c79\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:00.61734364Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50881"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:00.617413572Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"ba29c238-c0e1-4f6e-bfed-5ad768fc7c4b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50881","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:03.315309289Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"39.94388026s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.349239139Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.349446213Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.349458413Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.371901812Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.371958573Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.392322409Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.39236758Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.412771966Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.412834618Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.43258023Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.432634771Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.450632318Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.450673159Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.469174455Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.469216706Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.486383905Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.486427926Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.506489365Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.506552616Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.506838382Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.506982916Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.592186576Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.592256338Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.592263728Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.592270778Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.592277848Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.692722238Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.69716772Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.701753476Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.706256181Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.710830506Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.71580872Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.720637671Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.725294018Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.729744311Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:25.734503831Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:29.101918546Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:29.102243123Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:29.109415234Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:29.109537721Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:29.197813184Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:29.197972758Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\") failed: Property not found. retry in 44.809144ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:29.261655205Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\") failed: Property not found. retry in 45.022033ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:29.269523809Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:29.273120134Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:29.273299858Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\") failed: Property not found. retry in 117.759846ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:29.316277324Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\") failed: Property not found. retry in 141.284667ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:29.324471564Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:29.415354019Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:29.416317899Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\") failed: Property not found. retry in 234.741776ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:29.496680234Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\") failed: Property not found. retry in 116.831894ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:29.523017213Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\") failed: Property not found. retry in 51.933217ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:29.902220567Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\") failed: Property not found. retry in 386.680972ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:30.055617665Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\") failed: Property not found. retry in 430.872708ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:30.105875862Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\") failed: Property not found. retry in 119.466602ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:30.141883993Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\") failed: Property not found. retry in 56.818028ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:30.185473671Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\") failed: Property not found. retry in 77.929321ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:30.288952108Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\") failed: Property not found. retry in 171.719696ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:30.320982726Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\") failed: Property not found. retry in 283.363205ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:30.391187249Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\") failed: Property not found. retry in 60.894304ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:30.418999409Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\") failed: Property not found. retry in 529.119495ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:30.459934582Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\") failed: Property not found. retry in 90.483137ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:30.568395353Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\") failed: Property not found. retry in 930.477393ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:30.656975159Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:30.679431987Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50247"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:30.679494999Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"a71c6536-3975-45fb-b5bc-6ac39a8e64eb","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50247","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":145,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\") failed: Property not found. retry in 200.747265ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:30.687318652Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\") failed: Property not found. retry in 175.207925ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:30.717854498Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\") failed: Property not found. retry in 376.152254ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:30.757551656Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\") failed: Property not found. retry in 883.376348ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:30.955393599Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\") failed: Property not found. retry in 421.76841ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:31.053042535Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\") failed: Property not found. retry in 1.454115965s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:31.137330522Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\") failed: Property not found. retry in 809.419448ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:31.18378721Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\") failed: Property not found. retry in 686.508219ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:31.494941236Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\") failed: Property not found. retry in 1.331452041s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:31.614488028Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\") failed: Property not found. retry in 1.231696458s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:31.910248682Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\") failed: Property not found. retry in 1.190484953s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:32.005270443Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\") failed: Property not found. retry in 1.189841908s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:32.193155459Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\") failed: Property not found. retry in 526.830695ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:32.601030811Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\") failed: Property not found. retry in 850.669172ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:32.954812245Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\") failed: Property not found. retry in 780.571342ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:33.138064385Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\") failed: Property not found. retry in 753.426357ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:33.151048905Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\") failed: Property not found. retry in 828.015539ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:33.204372276Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\") failed: Property not found. retry in 859.645092ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:33.392886946Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\") failed: Property not found. retry in 770.975194ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:33.815936894Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\") failed: Property not found. retry in 824.872486ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:33.915649873Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\") failed: Property not found. retry in 1.021437238s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:33.930110484Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\") failed: Property not found. retry in 686.121517ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:34.045101941Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\") failed: Property not found. retry in 1.268735342s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:34.265100706Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\") failed: Property not found. retry in 819.505342ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:34.59776049Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\") failed: Property not found. retry in 1.488075789s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:34.741251091Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\") failed: Property not found. retry in 1.191325412s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:34.749592095Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\") failed: Property not found. retry in 1.446318385s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:34.963397702Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\") failed: Property not found. retry in 1.264425085s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:35.427862113Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\") failed: Property not found. retry in 670.927205ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:35.543152346Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\") failed: Property not found. retry in 837.593179ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:35.949496996Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\") failed: Property not found. retry in 955.451354ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:36.224993988Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\") failed: Property not found. retry in 723.71752ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:36.239597192Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64\") failed: Property not found. retry in 1.150313264s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:36.421127836Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\") failed: Property not found. retry in 702.113323ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:36.704055004Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373\") failed: Property not found. retry in 870.014498ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:36.79694859Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783\") failed: Property not found. retry in 1.090457721s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:36.972791315Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70\") failed: Property not found. retry in 1.228901376s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:37.19084494Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\") failed: Property not found. retry in 883.644067ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:37.416955753Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\") failed: Property not found. retry in 665.568264ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:38.311240973Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\") failed: Property not found. retry in 879.372833ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:38.987315786Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\") failed: Property not found. retry in 735.02101ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:39.87842305Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\") failed: Property not found. retry in 1.364544998s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:40.684460601Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\") failed: Property not found. retry in 1.481668075s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:42.06177869Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\") failed: Property not found. retry in 1.017522716s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:43.56059158Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\") failed: Property not found. retry in 1.011881352s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:44.606829458Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\") failed: Property not found. retry in 1.028055275s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:45.629122646Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107\") failed: Property not found. retry in 1.44693029s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:46.668085352Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:48.136770146Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_d4cfc513-67c2-47da-a859-e2a17e40cc64"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:48.136840567Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_b0eb1eaf-8367-49b6-a94f-86f250b89107"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:48.136846937Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_9dea0779-564e-4dbc-950b-f8c5b285e373"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:48.136852147Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_70034c43-bee0-4c56-bc50-cf8912a74f70"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:48.136860398Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_135b8c48-b910-4d26-b93c-acad7bbfa783"} {"msg":"Writing ledger to /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:48.137012421Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:48.138453521Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:48.140043354Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"88d23761-438e-4092-8a8d-540b4477763d","remote_addr":"[fd00:1122:3344:101::1]:62574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":22792333,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:48.171800265Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:54690"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::7]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:48.200571215Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3437"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:48.365061674Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3455"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:51.649831911Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3478"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:51.649947673Z","hostname":"buskin","pid":1590,"uri":"/cockroachdb","method":"POST","req_id":"8cd63a37-f8bd-4006-86ef-197820859ed9","remote_addr":"[fd00:1122:3344:101::1]:54690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3478041,"response_code":"204"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bff3110b-f67b-4955-a822-d477134772d8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 436d117c-1c16-43a3-b432-02e432890165, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 37798025-9c23-405f-8532-61a017646e4c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2bf3f193-0c63-460b-bad4-d0974b4b9606, kind: Service { id: 43617b58-d1ff-4c18-a5ff-af7f04834d55 }, name: Name(\"ntp-43617b58-d1ff-4c18-a5ff-af7f04834d55\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9dea0779-564e-4dbc-950b-f8c5b285e373, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: b0eb1eaf-8367-49b6-a94f-86f250b89107, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: d4cfc513-67c2-47da-a859-e2a17e40cc64, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70034c43-bee0-4c56-bc50-cf8912a74f70, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 135b8c48-b910-4d26-b93c-acad7bbfa783, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8bed16b-78d2-4e63-a1d0-74b2a84f550c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 931806a7-4f2a-4e78-88e7-dde35b44ffdc, kind: Service { id: b8bed16b-78d2-4e63-a1d0-74b2a84f550c }, name: Name(\"external-dns-b8bed16b-78d2-4e63-a1d0-74b2a84f550c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 129])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f900db17-3f82-488d-97bb-5e68315b969a, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: afdf6d97-9eaf-4d3d-84fd-ae0cd68cc463, kind: Service { id: f900db17-3f82-488d-97bb-5e68315b969a }, name: Name(\"external-dns-f900db17-3f82-488d-97bb-5e68315b969a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 253])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 26771b7d-eb9f-473a-8e8b-07e843276ba9, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0d75d2c6-30d2-4c9c-afdc-702493eb0827, kind: Service { id: 26771b7d-eb9f-473a-8e8b-07e843276ba9 }, name: Name(\"nexus-26771b7d-eb9f-473a-8e8b-07e843276ba9\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f358bb35-d0d7-4ff4-acab-03176c628de4, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: a022e989-05cf-4bcf-be27-4c24fd17399f, kind: Service { id: f358bb35-d0d7-4ff4-acab-03176c628de4 }, name: Name(\"nexus-f358bb35-d0d7-4ff4-acab-03176c628de4\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5427aa9d-f3ba-4d73-b918-7e9bd2515700, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 1cfb454f-c880-4dac-bec7-dc948725353d, kind: Service { id: 5427aa9d-f3ba-4d73-b918-7e9bd2515700 }, name: Name(\"nexus-5427aa9d-f3ba-4d73-b918-7e9bd2515700\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6e5ec7b4-5d6c-4258-a67a-de870d9b3e70, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 443ca187-01fc-48cb-8c40-fcf8671b1b47, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: cd84068e-4c88-4e7c-b752-937aeeea4eb8, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a59bf3ae-0bbe-4045-9561-e93a7dd9a912, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c9806c74-be17-49f6-a08b-7f1d1b3bcf6e, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: e6757d8a-0e5a-476e-ba5e-6eddb06637a3, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7643aa7-69d0-4e11-81f6-1e0195bf0112, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External } } } }, OmicronZoneConfig { id: 508f9d97-36bb-43a2-be97-9292c626ddc0, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8ddf5b4a-8de9-45ca-8c52-b491c6f6168a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External } } } }, OmicronZoneConfig { id: 781fb18a-8142-460f-97d3-6e8f870bb08f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 869a8107-be67-4970-bf87-7791ae7d8b68, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7673312c-f216-4503-9437-30203b65ec1d (zpool), kind: External } } } }, OmicronZoneConfig { id: 3bbf88dd-acaa-4a7c-b694-c3b7cef4d817, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4237ec86-747e-4b2e-a81d-30673b143f9c (zpool), kind: External } } } }, OmicronZoneConfig { id: 424e107e-83ee-477e-838f-5fd2d3f9faed, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0a8cb798-6ba7-43a1-aa2f-7aba85873942 (zpool), kind: External } } } }, OmicronZoneConfig { id: 830eb246-7c79-467d-b41c-ff29eef4ed9d, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e211498b-7438-4a66-89c4-ebf7a721e12d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:51.682601844Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:51.682944561Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:56743"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:51.683788329Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:51.683802239Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bff3110b-f67b-4955-a822-d477134772d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:51.707707347Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:51.707763169Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 436d117c-1c16-43a3-b432-02e432890165, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:51.726992839Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:51.7270483Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37798025-9c23-405f-8532-61a017646e4c, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:51.747861904Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:51.747913846Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9dea0779-564e-4dbc-950b-f8c5b285e373, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:51.768113536Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:51.768179798Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b0eb1eaf-8367-49b6-a94f-86f250b89107, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:51.788028942Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:51.788075923Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d4cfc513-67c2-47da-a859-e2a17e40cc64, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:51.80951821Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:51.809568991Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70034c43-bee0-4c56-bc50-cf8912a74f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:51.829820233Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:51.829869824Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 135b8c48-b910-4d26-b93c-acad7bbfa783, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8bed16b-78d2-4e63-a1d0-74b2a84f550c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:51.851092386Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:51.851142137Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8bed16b-78d2-4e63-a1d0-74b2a84f550c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f900db17-3f82-488d-97bb-5e68315b969a, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:51.934015934Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:51.934083906Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f900db17-3f82-488d-97bb-5e68315b969a, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 443ca187-01fc-48cb-8c40-fcf8671b1b47, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.010802765Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.010870056Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 443ca187-01fc-48cb-8c40-fcf8671b1b47, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6757d8a-0e5a-476e-ba5e-6eddb06637a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.088113457Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.088173148Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6757d8a-0e5a-476e-ba5e-6eddb06637a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4ab38590-bc9f-4317-9a2d-902570f44129 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7643aa7-69d0-4e11-81f6-1e0195bf0112, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.162808334Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.162868945Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7643aa7-69d0-4e11-81f6-1e0195bf0112, dataset_name: DatasetName { pool_name: ZpoolName { id: f2ad007c-d0d5-499a-a92a-f0e5d6ac476c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 508f9d97-36bb-43a2-be97-9292c626ddc0, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.233103359Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.23316349Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 508f9d97-36bb-43a2-be97-9292c626ddc0, dataset_name: DatasetName { pool_name: ZpoolName { id: eeebb608-8f09-47dd-a873-6005f1a33225 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ddf5b4a-8de9-45ca-8c52-b491c6f6168a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.309483121Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.309542872Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8ddf5b4a-8de9-45ca-8c52-b491c6f6168a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4917eed2-7e51-43de-bbf3-90e6e217a77b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 781fb18a-8142-460f-97d3-6e8f870bb08f, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.382161306Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.382223107Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 781fb18a-8142-460f-97d3-6e8f870bb08f, dataset_name: DatasetName { pool_name: ZpoolName { id: 137f7f84-2894-49f9-be7d-163348c068b9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 869a8107-be67-4970-bf87-7791ae7d8b68, dataset_name: DatasetName { pool_name: ZpoolName { id: 7673312c-f216-4503-9437-30203b65ec1d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.472224363Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.472285804Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 869a8107-be67-4970-bf87-7791ae7d8b68, dataset_name: DatasetName { pool_name: ZpoolName { id: 7673312c-f216-4503-9437-30203b65ec1d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3bbf88dd-acaa-4a7c-b694-c3b7cef4d817, dataset_name: DatasetName { pool_name: ZpoolName { id: 4237ec86-747e-4b2e-a81d-30673b143f9c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.539690759Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.53974397Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3bbf88dd-acaa-4a7c-b694-c3b7cef4d817, dataset_name: DatasetName { pool_name: ZpoolName { id: 4237ec86-747e-4b2e-a81d-30673b143f9c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 424e107e-83ee-477e-838f-5fd2d3f9faed, dataset_name: DatasetName { pool_name: ZpoolName { id: 0a8cb798-6ba7-43a1-aa2f-7aba85873942 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.61219431Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.612255822Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 424e107e-83ee-477e-838f-5fd2d3f9faed, dataset_name: DatasetName { pool_name: ZpoolName { id: 0a8cb798-6ba7-43a1-aa2f-7aba85873942 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 830eb246-7c79-467d-b41c-ff29eef4ed9d, dataset_name: DatasetName { pool_name: ZpoolName { id: e211498b-7438-4a66-89c4-ebf7a721e12d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.680356311Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.680409922Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 830eb246-7c79-467d-b41c-ff29eef4ed9d, dataset_name: DatasetName { pool_name: ZpoolName { id: e211498b-7438-4a66-89c4-ebf7a721e12d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.751146407Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.751221348Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.751492834Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.751687628Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.877620913Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.907886214Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.908111969Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.908134379Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.90815457Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.949795609Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.949997438Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.950019854Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.950053033Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:52.95006943Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.139436331Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.139675316Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.139708456Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.139769767Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.155813012Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.155940255Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.155950515Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.181325433Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.181371114Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.181937876Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.187094963Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.192393804Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.197681734Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.203408223Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.208854647Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.214209938Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.369807902Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.374621312Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.379498814Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.384445877Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.389582834Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.394706591Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.399667394Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.405754821Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.411643724Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.417944115Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.424745827Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.431410426Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.438612496Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.445194003Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.452058936Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.458832457Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.466224951Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.47285456Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.478987948Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.487452084Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.494987251Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.547069587Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.591333269Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.634630442Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.669449558Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.705345546Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.707023701Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.707038381Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.765256415Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 188, 89]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte1\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.765335796Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.769447202Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.769465892Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.835202773Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 204, 253]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte2\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.835265884Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.837662944Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.837680754Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.904582839Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 140, 17]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte3\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.90464252Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.908165743Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.908195394Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.96175718Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 233, 209]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte4\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.961797991Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.964347465Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:53.964368165Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.025895048Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 219, 129]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte5\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.025952819Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.028999152Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.029016553Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.029243297Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.029250517Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.029986763Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.029993163Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.030228948Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.030238358Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.030375801Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.030384231Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.035960137Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.038724955Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.038769166Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.0413668Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.04138723Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.043659728Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.077920602Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.082056958Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.084806816Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.084832526Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.146366148Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.210812562Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.216931169Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.223459595Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.230107644Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.237069209Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.244641667Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.25153488Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.261243753Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:11:54.269792241Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:11:58.625852758Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"81.977359523s"} {"msg":"Profile for oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.174714141Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.178733725Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.181125615Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.181156636Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.204348069Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.205652516Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.207925084Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.208103857Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.216051693Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.225630393Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.227002431Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.228507573Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.228527873Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.23749453Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.246463827Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.255608967Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.263066613Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.270376875Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.270426786Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.277930313Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.278933194Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.279169029Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.28702857Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.295037919Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.521326686Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.521697014Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.837691721Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56168"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.841738515Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"21dcdde1-cbff-4d58-ba44-fa274a866f4d","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56168","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3007,"response_code":"200"} {"msg":"Profile for oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.928115856Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:05.928290849Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817\") failed: Property not found. retry in 63.751413ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.095523919Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed\") failed: Property not found. retry in 71.939798ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.108362907Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a\") failed: Property not found. retry in 58.916575ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.120880838Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a\") failed: Property not found. retry in 55.108858ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.133290066Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e\") failed: Property not found. retry in 62.753074ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.144304866Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112\") failed: Property not found. retry in 37.426755ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.155000819Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912\") failed: Property not found. retry in 72.357802ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.165009447Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3\") failed: Property not found. retry in 67.81055ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.176475996Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d\") failed: Property not found. retry in 67.728826ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.188702551Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0\") failed: Property not found. retry in 37.80955ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.20019433Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70\") failed: Property not found. retry in 52.619142ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.211780032Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f\") failed: Property not found. retry in 72.335818ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.222881813Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a\") failed: Property not found. retry in 139.792148ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.23473521Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8\") failed: Property not found. retry in 52.77622ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.245224519Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a\") failed: Property not found. retry in 113.816026ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.256477633Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817\") failed: Property not found. retry in 88.703294ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.266677406Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed\") failed: Property not found. retry in 60.127504ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.277693106Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e\") failed: Property not found. retry in 67.161883ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.288051592Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112\") failed: Property not found. retry in 80.917535ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.296287653Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912\") failed: Property not found. retry in 132.94886ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.30475853Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3\") failed: Property not found. retry in 106.301829ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.315460943Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d\") failed: Property not found. retry in 81.686985ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.326084095Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0\") failed: Property not found. retry in 118.650546ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.337140785Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70\") failed: Property not found. retry in 66.075849ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.349802629Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f\") failed: Property not found. retry in 103.065477ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.360821348Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68\") failed: Property not found. retry in 68.204094ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.371180064Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8\") failed: Property not found. retry in 148.363091ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.382906429Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a\") failed: Property not found. retry in 112.22207ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.394833157Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817\") failed: Property not found. retry in 204.557358ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.408064573Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed\") failed: Property not found. retry in 229.297621ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.421234248Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e\") failed: Property not found. retry in 159.176923ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.433984584Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112\") failed: Property not found. retry in 219.655255ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.448508996Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912\") failed: Property not found. retry in 147.616745ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.460905485Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3\") failed: Property not found. retry in 231.423865ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.47122839Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d\") failed: Property not found. retry in 128.976472ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.489057501Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0\") failed: Property not found. retry in 291.136446ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.502369299Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70\") failed: Property not found. retry in 225.269571ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.513724016Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f\") failed: Property not found. retry in 104.24689ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.527719627Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68\") failed: Property not found. retry in 113.642257ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.53887097Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a\") failed: Property not found. retry in 281.833718ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.54945974Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8\") failed: Property not found. retry in 292.291252ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.560611063Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a\") failed: Property not found. retry in 378.97694ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.573086623Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e\") failed: Property not found. retry in 283.945624ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.607675504Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912\") failed: Property not found. retry in 473.83218ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.622116105Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817\") failed: Property not found. retry in 450.599946ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.632945931Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d\") failed: Property not found. retry in 440.819399ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.644529632Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f\") failed: Property not found. retry in 554.700857ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.65640492Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68\") failed: Property not found. retry in 274.054525ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.667296177Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed\") failed: Property not found. retry in 404.385515ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.677951179Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112\") failed: Property not found. retry in 597.843146ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.688735874Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:07.689199743Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:07.689382987Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3\") failed: Property not found. retry in 293.883795ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.715554363Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70\") failed: Property not found. retry in 302.74389ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.752775549Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0\") failed: Property not found. retry in 561.158148ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.808463999Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a\") failed: Property not found. retry in 341.604247ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.845361559Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8\") failed: Property not found. retry in 235.566017ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.866279155Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e\") failed: Property not found. retry in 461.991855ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.906319019Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4\") failed: Property not found. retry in 31.783313ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.939160724Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68\") failed: Property not found. retry in 422.512504ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.955733139Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a\") failed: Property not found. retry in 1.053355622s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.966934513Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4\") failed: Property not found. retry in 114.989182ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:07.981872664Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3\") failed: Property not found. retry in 888.90907ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:08.022896639Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70\") failed: Property not found. retry in 609.093931ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:08.071585064Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed\") failed: Property not found. retry in 1.000829654s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:08.098075036Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817\") failed: Property not found. retry in 1.053423299s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:08.111009856Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4\") failed: Property not found. retry in 173.12566ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:08.123324512Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912\") failed: Property not found. retry in 1.18412753s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:08.138943098Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d\") failed: Property not found. retry in 729.808453ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:08.152657594Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8\") failed: Property not found. retry in 1.118276796s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:08.166997942Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a\") failed: Property not found. retry in 656.624716ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:08.201848609Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f\") failed: Property not found. retry in 1.094887045s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:08.228350731Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112\") failed: Property not found. retry in 740.423968ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:08.301780212Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4\") failed: Property not found. retry in 556.359748ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:08.316449758Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e\") failed: Property not found. retry in 1.007857961s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:08.383533316Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0\") failed: Property not found. retry in 744.037259ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:08.396299612Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68\") failed: Property not found. retry in 971.958151ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:08.410505668Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70\") failed: Property not found. retry in 1.084365083s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:08.69653319Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a\") failed: Property not found. retry in 1.099097891s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:08.886633452Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4\") failed: Property not found. retry in 436.382999ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:08.907693102Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d\") failed: Property not found. retry in 1.287392687s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:08.9589469Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3\") failed: Property not found. retry in 1.066421408s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:09.007064473Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a\") failed: Property not found. retry in 1.073659943s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:09.101229336Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed\") failed: Property not found. retry in 1.231295308s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:09.141490005Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112\") failed: Property not found. retry in 625.221915ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:09.284681299Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817\") failed: Property not found. retry in 637.89516ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:09.371507689Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4\") failed: Property not found. retry in 848.561643ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:09.423973452Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912\") failed: Property not found. retry in 668.245961ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:09.55676601Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0\") failed: Property not found. retry in 603.268983ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:09.570525227Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f\") failed: Property not found. retry in 941.580136ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:09.586445379Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68\") failed: Property not found. retry in 952.827619ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:09.601296568Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8\") failed: Property not found. retry in 731.580641ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:09.61621579Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e\") failed: Property not found. retry in 837.733835ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:09.628956005Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70\") failed: Property not found. retry in 976.069841ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:09.795269972Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112\") failed: Property not found. retry in 1.422555297s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:09.924168909Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a\") failed: Property not found. retry in 633.230736ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:10.001175394Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817\") failed: Property not found. retry in 1.408795287s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:10.022550649Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3\") failed: Property not found. retry in 785.658988ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:10.087638876Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0\") failed: Property not found. retry in 1.248653404s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:10.187694451Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a\") failed: Property not found. retry in 1.215968185s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:10.199243772Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912\") failed: Property not found. retry in 1.380909036s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:10.238171763Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d\") failed: Property not found. retry in 904.481959ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:10.258430726Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4\") failed: Property not found. retry in 733.90325ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:10.28646193Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8\") failed: Property not found. retry in 1.492047519s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:10.360866521Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed\") failed: Property not found. retry in 1.01150054s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:10.385462433Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e\") failed: Property not found. retry in 1.179484325s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:10.476355448Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f\") failed: Property not found. retry in 1.049249402s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:10.537777948Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68\") failed: Property not found. retry in 851.711412ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:10.56568083Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a\") failed: Property not found. retry in 1.079854259s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:10.644247578Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70\") failed: Property not found. retry in 647.824131ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:10.77964039Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3\") failed: Property not found. retry in 885.519383ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:10.882498544Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4\") failed: Property not found. retry in 1.100151736s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:11.030495479Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d\") failed: Property not found. retry in 586.262049ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:11.17689292Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112\") failed: Property not found. retry in 1.091170996s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:11.36302271Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed\") failed: Property not found. retry in 1.471830078s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:11.411868288Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a\") failed: Property not found. retry in 530.812413ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:11.429176668Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70\") failed: Property not found. retry in 730.757903ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:11.44073542Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68\") failed: Property not found. retry in 1.134278619s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:11.451667197Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817\") failed: Property not found. retry in 698.790535ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:11.462507943Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0\") failed: Property not found. retry in 1.213080714s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:11.476443924Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f\") failed: Property not found. retry in 847.624691ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:11.601120803Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912\") failed: Property not found. retry in 583.333661ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:11.631447905Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e\") failed: Property not found. retry in 1.219908153s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:11.667594378Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a\") failed: Property not found. retry in 1.305441029s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:11.738099508Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d\") failed: Property not found. retry in 964.641495ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:11.77512343Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3\") failed: Property not found. retry in 561.615089ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:11.78521161Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8\") failed: Property not found. retry in 1.491583662s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:11.864430701Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a\") failed: Property not found. retry in 1.398182362s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:11.972551605Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4\") failed: Property not found. retry in 1.023255789s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:12.14311469Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817\") failed: Property not found. retry in 1.397111104s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:12.172067923Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70\") failed: Property not found. retry in 1.154524155s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:12.183062073Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912\") failed: Property not found. retry in 908.694345ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:12.226273623Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3\") failed: Property not found. retry in 1.199802449s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:12.357052369Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f\") failed: Property not found. retry in 715.914908ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:12.459202389Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112\") failed: Property not found. retry in 957.444359ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:12.468836639Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68\") failed: Property not found. retry in 1.301455047s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:12.596431749Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0\") failed: Property not found. retry in 620.811452ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:12.701773975Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d\") failed: Property not found. retry in 1.239067852s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:12.758934966Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed\") failed: Property not found. retry in 740.343635ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:12.895150345Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e\") failed: Property not found. retry in 1.093863944s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:12.904687804Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:12.935523367Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:12.936393155Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a\") failed: Property not found. retry in 756.47795ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:13.056324275Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912\") failed: Property not found. retry in 1.016255488s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:13.147854243Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4\") failed: Property not found. retry in 640.386906ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:13.179316008Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c\") failed: Property not found. retry in 65.7227ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:13.188573821Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f\") failed: Property not found. retry in 1.128514778s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:13.198627461Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c\") failed: Property not found. retry in 111.840149ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:13.274534943Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0\") failed: Property not found. retry in 1.343521427s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:13.352077849Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70\") failed: Property not found. retry in 1.036325029s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:13.371352001Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8\") failed: Property not found. retry in 986.480669ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:13.387252852Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a\") failed: Property not found. retry in 1.29738353s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:13.400537119Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c\") failed: Property not found. retry in 169.77639ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:13.413120541Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112\") failed: Property not found. retry in 552.089216ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:13.437219414Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3\") failed: Property not found. retry in 762.710569ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:13.567557351Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817\") failed: Property not found. retry in 1.132545273s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:13.579089111Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c\") failed: Property not found. retry in 334.881173ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:13.592319247Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed\") failed: Property not found. retry in 510.014357ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:13.645358833Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:13.804610542Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:13.805067702Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:13.805237745Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:13.810892313Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a\") failed: Property not found. retry in 709.461835ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:13.826153111Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4\") failed: Property not found. retry in 1.064626083s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:13.836790093Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68\") failed: Property not found. retry in 707.706283ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:13.910690563Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c\") failed: Property not found. retry in 409.124235ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:13.938702247Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112\") failed: Property not found. retry in 662.476008ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.000543386Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e\") failed: Property not found. retry in 1.355915758s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.010475263Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d\") failed: Property not found. retry in 733.035482ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.021660626Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9\") failed: Property not found. retry in 56.111815ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.032180876Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700\") failed: Property not found. retry in 27.348754ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.042091842Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700\") failed: Property not found. retry in 59.487371ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.080554704Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9\") failed: Property not found. retry in 86.27503ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.106551466Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700\") failed: Property not found. retry in 191.178907ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.162167145Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed\") failed: Property not found. retry in 801.178623ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.175657926Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912\") failed: Property not found. retry in 1.051005966s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.197691205Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9\") failed: Property not found. retry in 130.153912ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.218092Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f\") failed: Property not found. retry in 1.351313036s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.338064251Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3\") failed: Property not found. retry in 532.488012ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.355500584Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9\") failed: Property not found. retry in 482.413124ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.374429769Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c\") failed: Property not found. retry in 738.327037ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.388420491Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700\") failed: Property not found. retry in 505.040225ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.403687119Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8\") failed: Property not found. retry in 688.234363ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.41621957Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70\") failed: Property not found. retry in 887.429829ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.440348163Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a\") failed: Property not found. retry in 1.313915899s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.574375067Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68\") failed: Property not found. retry in 1.003310247s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.635666824Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112\") failed: Property not found. retry in 1.13671054s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.675428153Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0\") failed: Property not found. retry in 1.465669365s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.715758104Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a\") failed: Property not found. retry in 922.11005ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.738695242Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817\") failed: Property not found. retry in 731.670649ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.751154102Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d\") failed: Property not found. retry in 848.025986ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.764875877Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9\") failed: Property not found. retry in 465.141222ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.871575722Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3\") failed: Property not found. retry in 610.174314ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.900735769Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4\") failed: Property not found. retry in 1.262764594s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.915786473Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700\") failed: Property not found. retry in 700.218075ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.926973986Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed\") failed: Property not found. retry in 560.491528ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:14.98758472Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8\") failed: Property not found. retry in 1.452240101s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:15.115159109Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c\") failed: Property not found. retry in 1.282312509s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:15.135802459Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912\") failed: Property not found. retry in 665.110242ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:15.257767011Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70\") failed: Property not found. retry in 1.443205716s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:15.337307389Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9\") failed: Property not found. retry in 1.440951228s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:15.345369637Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e\") failed: Property not found. retry in 762.954324ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:15.376347123Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817\") failed: Property not found. retry in 548.897915ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:15.505177108Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3\") failed: Property not found. retry in 1.319067923s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:15.520481707Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed\") failed: Property not found. retry in 1.320539286s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:15.55851227Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d\") failed: Property not found. retry in 599.870794ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:15.648931415Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700\") failed: Property not found. retry in 576.603705ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:15.663850576Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68\") failed: Property not found. retry in 1.394745147s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:15.674458677Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a\") failed: Property not found. retry in 829.305862ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:15.682924573Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f\") failed: Property not found. retry in 666.173434ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:15.69956105Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:15.79693785Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:15.797187225Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112\") failed: Property not found. retry in 979.442535ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:15.828094369Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a\") failed: Property not found. retry in 946.902828ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:15.902170674Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912\") failed: Property not found. retry in 543.167567ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:15.935474248Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47\") failed: Property not found. retry in 31.133525ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:16.006483618Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47\") failed: Property not found. retry in 65.542375ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:16.050136278Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817\") failed: Property not found. retry in 1.085298459s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:16.06559072Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47\") failed: Property not found. retry in 120.542687ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:16.124781444Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e\") failed: Property not found. retry in 1.31332658s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:16.150625452Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4\") failed: Property not found. retry in 633.079452ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:16.188774947Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0\") failed: Property not found. retry in 1.039338291s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:16.197306425Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700\") failed: Property not found. retry in 849.727315ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:16.24985296Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47\") failed: Property not found. retry in 573.339083ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:16.260168715Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d\") failed: Property not found. retry in 723.382757ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:16.267931207Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f\") failed: Property not found. retry in 1.318301595s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:16.378355739Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c\") failed: Property not found. retry in 945.877755ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:16.444347814Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912\") failed: Property not found. retry in 1.432666767s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:16.49306253Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a\") failed: Property not found. retry in 1.309935818s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:16.52474439Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8\") failed: Property not found. retry in 630.712001ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:16.580101984Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70\") failed: Property not found. retry in 593.579996ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:16.795104966Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9\") failed: Property not found. retry in 511.574811ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:16.806889421Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112\") failed: Property not found. retry in 579.450629ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:16.82026375Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4\") failed: Property not found. retry in 752.660904ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:16.833916924Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47\") failed: Property not found. retry in 512.359498ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:16.845510906Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3\") failed: Property not found. retry in 1.160105538s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:16.855951654Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a\") failed: Property not found. retry in 807.021803ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:16.867437033Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed\") failed: Property not found. retry in 749.867745ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:16.891972575Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d\") failed: Property not found. retry in 696.247219ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:17.007694157Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68\") failed: Property not found. retry in 1.099179871s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:17.082947326Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700\") failed: Property not found. retry in 648.90616ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:17.112939871Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817\") failed: Property not found. retry in 807.955688ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:17.163981124Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8\") failed: Property not found. retry in 1.39218596s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:17.229419018Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0\") failed: Property not found. retry in 673.7916ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:17.254581462Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9\") failed: Property not found. retry in 1.382769574s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:17.355154109Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47\") failed: Property not found. retry in 620.635564ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:17.378439404Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70\") failed: Property not found. retry in 881.208481ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:17.406120571Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112\") failed: Property not found. retry in 572.347826ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:17.420779367Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c\") failed: Property not found. retry in 1.192020698s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:17.435063385Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e\") failed: Property not found. retry in 1.234065224s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:17.479547182Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4\") failed: Property not found. retry in 1.216337024s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:17.625600106Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed\") failed: Property not found. retry in 592.612259ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:17.660699198Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a\") failed: Property not found. retry in 1.473329284s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:17.695334479Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f\") failed: Property not found. retry in 1.289827688s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:17.713838235Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d\") failed: Property not found. retry in 695.005103ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:17.733504055Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700\") failed: Property not found. retry in 1.09204212s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:17.784940427Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a\") failed: Property not found. retry in 1.087338647s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:17.848974052Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912\") failed: Property not found. retry in 1.363179251s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:17.939444398Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0\") failed: Property not found. retry in 1.084361839s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:17.951840466Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817\") failed: Property not found. retry in 1.049469782s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:17.987253084Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112\") failed: Property not found. retry in 590.702994ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:18.010927508Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47\") failed: Property not found. retry in 538.082886ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:18.034630272Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3\") failed: Property not found. retry in 1.053124118s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:18.052466674Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68\") failed: Property not found. retry in 1.355589429s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:18.197075748Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed\") failed: Property not found. retry in 622.685565ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:18.267521736Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70\") failed: Property not found. retry in 869.20852ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:18.302581967Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d\") failed: Property not found. retry in 1.433881141s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:18.442430892Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47\") failed: Property not found. retry in 897.008917ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:18.585515574Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112\") failed: Property not found. retry in 701.740336ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:18.613687312Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8\") failed: Property not found. retry in 797.337843ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:18.632393402Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c\") failed: Property not found. retry in 873.878768ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:18.643458172Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e\") failed: Property not found. retry in 845.923995ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:18.725607505Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9\") failed: Property not found. retry in 1.335436589s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:18.750209717Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4\") failed: Property not found. retry in 673.026075ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:18.853735395Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700\") failed: Property not found. retry in 511.411361ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:18.887411987Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed\") failed: Property not found. retry in 614.866145ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:18.900111342Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a\") failed: Property not found. retry in 628.737744ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:18.948148403Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f\") failed: Property not found. retry in 1.260360958s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:19.015399935Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0\") failed: Property not found. retry in 583.19014ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:19.048152708Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817\") failed: Property not found. retry in 864.297465ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:19.059250709Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3\") failed: Property not found. retry in 1.115226991s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:19.117974353Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a\") failed: Property not found. retry in 1.208525414s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:19.181481616Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70\") failed: Property not found. retry in 870.74297ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:19.193938366Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912\") failed: Property not found. retry in 1.310322852s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:19.316121233Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112\") failed: Property not found. retry in 1.024826711s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:19.328437759Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700\") failed: Property not found. retry in 1.369439319s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:19.409484059Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8\") failed: Property not found. retry in 744.740829ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:19.439563506Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47\") failed: Property not found. retry in 518.560577ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:19.500393844Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed\") failed: Property not found. retry in 1.474284033s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:19.531920481Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4\") failed: Property not found. retry in 768.641187ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:19.545812101Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c\") failed: Property not found. retry in 868.903542ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:19.559730361Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68\") failed: Property not found. retry in 855.370506ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:19.574033639Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e\") failed: Property not found. retry in 820.329324ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:19.585080699Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a\") failed: Property not found. retry in 1.448207427s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:19.597836495Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0\") failed: Property not found. retry in 1.079994737s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:19.644903276Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d\") failed: Property not found. retry in 618.199591ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:19.889808451Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817\") failed: Property not found. retry in 799.430516ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:19.937417173Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47\") failed: Property not found. retry in 973.937327ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:20.033274182Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70\") failed: Property not found. retry in 1.393921459s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:20.082782983Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9\") failed: Property not found. retry in 1.222072361s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:20.100923172Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8\") failed: Property not found. retry in 855.266908ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:20.197511274Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3\") failed: Property not found. retry in 775.478563ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:20.252282016Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f\") failed: Property not found. retry in 872.316694ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:20.289030102Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4\") failed: Property not found. retry in 1.488000527s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:20.32729732Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112\") failed: Property not found. retry in 1.02470628s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:20.365736641Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a\") failed: Property not found. retry in 1.476583737s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:20.400478765Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e\") failed: Property not found. retry in 897.589916ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:20.417390708Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c\") failed: Property not found. retry in 583.139252ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:20.442840108Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68\") failed: Property not found. retry in 798.186531ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:20.460101308Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d\") failed: Property not found. retry in 671.992919ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:20.522738134Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912\") failed: Property not found. retry in 1.298753203s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:20.641354596Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0\") failed: Property not found. retry in 1.209047825s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:20.739072473Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817\") failed: Property not found. retry in 552.134794ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:20.754640427Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700\") failed: Property not found. retry in 749.306931ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:20.793641611Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed\") failed: Property not found. retry in 1.486548255s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:21.023324558Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47\") failed: Property not found. retry in 1.125782172s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:21.040462795Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3\") failed: Property not found. retry in 751.944874ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:21.055414117Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c\") failed: Property not found. retry in 1.011880621s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:21.070161534Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8\") failed: Property not found. retry in 799.833882ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:21.092871828Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a\") failed: Property not found. retry in 700.077638ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:21.110685239Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f\") failed: Property not found. retry in 1.210848823s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:21.175852997Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d\") failed: Property not found. retry in 561.748113ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:21.209691572Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68\") failed: Property not found. retry in 1.088192361s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:21.273178546Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817\") failed: Property not found. retry in 1.026855451s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:21.321616385Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e\") failed: Property not found. retry in 1.269442409s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:21.333677457Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9\") failed: Property not found. retry in 1.009915743s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:21.343873439Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112\") failed: Property not found. retry in 966.303989ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:21.405137496Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70\") failed: Property not found. retry in 1.370434311s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:21.49360411Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700\") failed: Property not found. retry in 842.852636ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:21.555733105Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d\") failed: Property not found. retry in 977.084097ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:21.783712377Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3\") failed: Property not found. retry in 1.116567341s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:21.817712776Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a\") failed: Property not found. retry in 1.09443747s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:21.830501973Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4\") failed: Property not found. retry in 566.188366ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:21.842489773Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a\") failed: Property not found. retry in 542.263053ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:21.889023392Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c\") failed: Property not found. retry in 1.426429011s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:22.100780666Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47\") failed: Property not found. retry in 1.084165691s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:22.187270009Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817\") failed: Property not found. retry in 1.234429952s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:22.36627786Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9\") failed: Property not found. retry in 786.368123ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:22.378501025Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68\") failed: Property not found. retry in 1.011027869s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:22.391657999Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700\") failed: Property not found. retry in 924.925459ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:22.484151947Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3\") failed: Property not found. retry in 1.006553172s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:22.963911147Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9\") failed: Property not found. retry in 806.847193ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:23.187000257Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47\") failed: Property not found. retry in 554.419691ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:23.296967339Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700\") failed: Property not found. retry in 727.840538ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:23.468888283Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47\") failed: Property not found. retry in 576.624256ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:23.876851856Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.515797335Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_external_dns_f900db17-3f82-488d-97bb-5e68315b969a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.515857096Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_cd84068e-4c88-4e7c-b752-937aeeea4eb8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.515864146Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_8ddf5b4a-8de9-45ca-8c52-b491c6f6168a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.515869856Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_3bbf88dd-acaa-4a7c-b694-c3b7cef4d817"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.515879146Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_26771b7d-eb9f-473a-8e8b-07e843276ba9"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.515885016Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_424e107e-83ee-477e-838f-5fd2d3f9faed"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.515890677Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_c9806c74-be17-49f6-a08b-7f1d1b3bcf6e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.515896566Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_clickhouse_443ca187-01fc-48cb-8c40-fcf8671b1b47"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.515902047Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_d7643aa7-69d0-4e11-81f6-1e0195bf0112"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.515908257Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_f358bb35-d0d7-4ff4-acab-03176c628de4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.515913837Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_a59bf3ae-0bbe-4045-9561-e93a7dd9a912"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.515919157Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_e6757d8a-0e5a-476e-ba5e-6eddb06637a3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.515926077Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_830eb246-7c79-467d-b41c-ff29eef4ed9d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.515931887Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_external_dns_b8bed16b-78d2-4e63-a1d0-74b2a84f550c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.515937927Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_508f9d97-36bb-43a2-be97-9292c626ddc0"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.515943018Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_oximeter_6e5ec7b4-5d6c-4258-a67a-de870d9b3e70"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.515951728Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_5427aa9d-f3ba-4d73-b918-7e9bd2515700"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.515969328Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_781fb18a-8142-460f-97d3-6e8f870bb08f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.515975048Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_869a8107-be67-4970-bf87-7791ae7d8b68"} {"msg":"Writing ledger to /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.518279226Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.526527858Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.534597466Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"7c97e184-6d80-4b0f-a1b8-8c6938a70e8b","remote_addr":"[fd00:1122:3344:101::1]:56743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":32850959,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.566052802Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1212"} {"msg":"Writing ledger to /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.567142765Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.569218848Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.575324345Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:679"} {"msg":"Nexus address: [fd00:1122:3344:101::a]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.5827576Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:692"} {"msg":"rack_network_config: RackNetworkConfigV1 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV1 {\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.1.174,\n prefix: 24,\n },\n ),\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.1.100,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.677197269Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:803"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2f1c0d8e-5768-4d21-9912-5823b0779f9c/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:24.948755369Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2f1c0d8e-5768-4d21-9912-5823b0779f9c/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:25.059396606Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2f1c0d8e-5768-4d21-9912-5823b0779f9c/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:25.143827005Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2f1c0d8e-5768-4d21-9912-5823b0779f9c/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:25.220190357Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2f1c0d8e-5768-4d21-9912-5823b0779f9c/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:25.407588833Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2f1c0d8e-5768-4d21-9912-5823b0779f9c/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:25.54515885Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2f1c0d8e-5768-4d21-9912-5823b0779f9c/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:25.857376418Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2f1c0d8e-5768-4d21-9912-5823b0779f9c/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:26.195783901Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2f1c0d8e-5768-4d21-9912-5823b0779f9c/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:26.46818343Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:26.741093558Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T05:12:26.741491216Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2f1c0d8e-5768-4d21-9912-5823b0779f9c/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:26.97339764Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2f1c0d8e-5768-4d21-9912-5823b0779f9c/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:27.623823748Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T05:12:27.746197178Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2f1c0d8e-5768-4d21-9912-5823b0779f9c/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:28.30472731Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:28.508354255Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T05:12:28.748830577Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2f1c0d8e-5768-4d21-9912-5823b0779f9c/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:28.798142965Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2f1c0d8e-5768-4d21-9912-5823b0779f9c/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:29.611720163Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T05:12:29.752225992Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:30.507865472Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T05:12:30.755930613Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2f1c0d8e-5768-4d21-9912-5823b0779f9c/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:30.89924664Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T05:12:31.760184516Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2f1c0d8e-5768-4d21-9912-5823b0779f9c/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:31.859486786Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:32.508110865Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T05:12:32.763432387Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2f1c0d8e-5768-4d21-9912-5823b0779f9c/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:32.941378596Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T05:12:33.766615158Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/83429885-0785-47bc-b249-4b860f5e232d): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:12:34.508553612Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6e418489-e26b-42bf-828b-0f9702c3f0a8\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:12:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6e418489-e26b-42bf-828b-0f9702c3f0a8\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:35.90168972Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:36.010881186Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37910"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:36.012690704Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"878a4cf4-00ab-4754-ae2a-633d95be1859","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37910","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1031,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a9c64630-31f7-44b2-ad33-9e5434f45dab\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 05:12:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a9c64630-31f7-44b2-ad33-9e5434f45dab\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:37.358106927Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:38.580357824Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:38.742596266Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:51359"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:38.744360533Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:38.837823621Z","hostname":"buskin","pid":1590,"uri":"/inventory","method":"GET","req_id":"b1c73a77-fa7a-414a-91dd-e20d9be9b1d4","remote_addr":"[fd00:1122:3344:101::b]:51359","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":95113,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:38.838713249Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:38.838885433Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:38.845131383Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:38.845811167Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"GET","req_id":"b821674b-9c1c-4aef-bc2a-8fbe32b10912","remote_addr":"[fd00:1122:3344:101::b]:51359","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7509,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:39.165603813Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:58522"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:39.167710997Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"ac4fd02f-7190-45b5-9889-17dedc0bac50","remote_addr":"[fd00:1122:3344:101::a]:58522","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":2087,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:39.881977385Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:51103"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:39.887649303Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 931806a7-4f2a-4e78-88e7-dde35b44ffdc, kind: Service { id: b8bed16b-78d2-4e63-a1d0-74b2a84f550c }, name: Name(\"external-dns-b8bed16b-78d2-4e63-a1d0-74b2a84f550c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 129])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: afdf6d97-9eaf-4d3d-84fd-ae0cd68cc463, kind: Service { id: f900db17-3f82-488d-97bb-5e68315b969a }, name: Name(\"external-dns-f900db17-3f82-488d-97bb-5e68315b969a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 253])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0d75d2c6-30d2-4c9c-afdc-702493eb0827, kind: Service { id: 26771b7d-eb9f-473a-8e8b-07e843276ba9 }, name: Name(\"nexus-26771b7d-eb9f-473a-8e8b-07e843276ba9\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 1cfb454f-c880-4dac-bec7-dc948725353d, kind: Service { id: 5427aa9d-f3ba-4d73-b918-7e9bd2515700 }, name: Name(\"nexus-5427aa9d-f3ba-4d73-b918-7e9bd2515700\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: a022e989-05cf-4bcf-be27-4c24fd17399f, kind: Service { id: f358bb35-d0d7-4ff4-acab-03176c628de4 }, name: Name(\"nexus-f358bb35-d0d7-4ff4-acab-03176c628de4\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:39.887815237Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:39.888083762Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:39.888096923Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:39.888131823Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:39.888169874Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:39.888195555Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:39.888207185Z","hostname":"buskin","pid":1590,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"e92a9c02-fb8f-4dfe-bc53-7125a493b853","remote_addr":"[fd00:1122:3344:101::a]:51103","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":5811,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:39.890595625Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:865"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:40.09951993Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:56354"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:40.100446319Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:40.190069807Z","hostname":"buskin","pid":1590,"uri":"/inventory","method":"GET","req_id":"f361cd4a-6808-4600-8bb8-90f018232704","remote_addr":"[fd00:1122:3344:101::a]:56354","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":90466,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:40.190531086Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:40.190597627Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:40.190910964Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:12:40.19117491Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"GET","req_id":"c447182b-c15b-409b-8d1e-2b830e98df63","remote_addr":"[fd00:1122:3344:101::a]:56354","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":771,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:04.997881597Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:54648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:05.374998438Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"bd9a5a36-c435-4d3a-93f5-57fd276c1d6a","remote_addr":"[fd00:1122:3344:101::b]:54648","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":377071,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:05.375187052Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:43745"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:05.37895777Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"506c3779-b8f6-42ba-bb3f-5799b54bb0b8","remote_addr":"[fd00:1122:3344:101::b]:54648","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3568,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:05.42261332Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:59420"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:05.439666716Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:05.43986278Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:05.453956904Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:05.455028826Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:05.455251041Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"PUT","req_id":"ee62fd9d-55bb-4e67-8ae7-6a7b72c24f68","remote_addr":"[fd00:1122:3344:101::b]:54648","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":27330,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:05.478775411Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"8adc61f9-369e-4ddb-bf46-fd5789640b86","remote_addr":"[fd00:1122:3344:101::a]:43745","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":103523,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:05.483361497Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"b633b546-90ab-450f-8ca6-cf94e6760168","remote_addr":"[fd00:1122:3344:101::a]:43745","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1107,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:05.591018201Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"b27916a9-3783-4e33-8456-7d130d2e702b","remote_addr":"[fd00:1122:3344:101::c]:59420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":168266,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:05.591417819Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"898fcbc1-5942-4711-ba1f-41fda9dbf082","remote_addr":"[fd00:1122:3344:101::c]:59420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:06.054269537Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41800"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:06.054363139Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"fec7cd4d-eacb-4fa5-81d8-7a2a9abd9930","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41800","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:35.176360029Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:49054"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:35.231134261Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"f7f364e8-0d7e-4044-b2b9-eea73d05222e","remote_addr":"[fd00:1122:3344:101::b]:49054","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":54710,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:35.231542789Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"4326686a-a47c-4bf6-9579-f6c559e249d0","remote_addr":"[fd00:1122:3344:101::b]:49054","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":131,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:35.517300086Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:53354"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:35.56882668Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"057396ae-69cd-4014-9856-8582371e57e9","remote_addr":"[fd00:1122:3344:101::a]:53354","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":51484,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:35.569593946Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"dc2e99e4-6467-4025-a7e1-0c7bb43d0a83","remote_addr":"[fd00:1122:3344:101::a]:53354","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:35.572929725Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:61226"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:35.623225724Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"15a1be78-34ba-4903-9efc-36c7d40d8632","remote_addr":"[fd00:1122:3344:101::c]:61226","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50199,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:35.62354177Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"29e89f79-5424-44ea-8d89-16ceb8970e0c","remote_addr":"[fd00:1122:3344:101::c]:61226","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:36.109202283Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63938"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:36.109702884Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"ac7c6031-1274-4100-9f9f-4de1d0590522","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63938","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":246,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:49.011601536Z","hostname":"buskin","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:50.744815373Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:48878","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::d]:57308"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:13:50.759408967Z","hostname":"buskin","pid":1590,"uri":"//83429885-0785-47bc-b249-4b860f5e232d","method":"GET","req_id":"47533008-89d4-4771-a4dd-ebfdd50f40b2","remote_addr":"[fd00:1122:3344:101::d]:57308","local_addr":"[fd00:1122:3344:101::1]:48878","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":12103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:14:05.014418773Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:39640"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:14:05.063504327Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"fc55df0f-fc62-4963-bcba-a2e757ebcfba","remote_addr":"[fd00:1122:3344:101::b]:39640","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49024,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:14:05.063770022Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"fe8c09ca-6ff1-468d-813d-3df6cd223dc9","remote_addr":"[fd00:1122:3344:101::b]:39640","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:14:05.280615102Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:36004"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:14:05.332472943Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"04f3db47-1202-46a3-b121-b4c71703d32f","remote_addr":"[fd00:1122:3344:101::a]:36004","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":51754,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:14:05.332725188Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"f2ea22fd-9616-4397-9af2-524fbbc40395","remote_addr":"[fd00:1122:3344:101::a]:36004","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:14:05.412914789Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:45585"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:14:05.461561753Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"e55a7b6b-e010-4f36-ae47-1bdd863f9261","remote_addr":"[fd00:1122:3344:101::c]:45585","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48553,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:14:05.461848959Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"c52695c7-e2cb-40f1-bd48-38450c90a122","remote_addr":"[fd00:1122:3344:101::c]:45585","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:14:06.145575991Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42942"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:14:06.145702374Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"eb0cc504-197d-40c7-bc3a-927778355772","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42942","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":104,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:14:20.744956595Z","hostname":"buskin","pid":1590,"uri":"//83429885-0785-47bc-b249-4b860f5e232d","method":"GET","req_id":"d2227a17-179c-4407-b4b4-d20f8e23cc74","remote_addr":"[fd00:1122:3344:101::d]:57308","local_addr":"[fd00:1122:3344:101::1]:48878","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":636,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:14:35.145363631Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:35033"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:14:35.197329645Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"237a1881-8245-4e27-bb69-f9350622bc17","remote_addr":"[fd00:1122:3344:101::b]:35033","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":51891,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:14:35.197574659Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"fdad1c46-5ff2-48ce-9bb1-ad49366e0e9e","remote_addr":"[fd00:1122:3344:101::b]:35033","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":92,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:14:35.354754666Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:33372"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:14:35.404296168Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"400f7bbe-9326-4878-b1f4-f4fa95032042","remote_addr":"[fd00:1122:3344:101::a]:33372","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49433,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:14:35.405154966Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"b265869e-a258-4614-a1bd-793affeb16a0","remote_addr":"[fd00:1122:3344:101::a]:33372","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":708,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:14:35.439227667Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:38764"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:14:35.48784928Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"99e6556c-64fb-47e9-81f6-9d1d2f7518d5","remote_addr":"[fd00:1122:3344:101::c]:38764","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48534,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:14:35.488106355Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"9e3112bd-8cc8-43bd-8d82-308a9dfd7955","remote_addr":"[fd00:1122:3344:101::c]:38764","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":92,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:14:36.179437215Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:14:36.179574288Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"aa5a0e74-d5a1-45b5-9306-8ab233701e82","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44177","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":72,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:14:50.745457513Z","hostname":"buskin","pid":1590,"uri":"//83429885-0785-47bc-b249-4b860f5e232d","method":"GET","req_id":"f1f61ad8-d892-49d3-be3f-01e374e10043","remote_addr":"[fd00:1122:3344:101::d]:57308","local_addr":"[fd00:1122:3344:101::1]:48878","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":576,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:15:04.941129453Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:51324"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:15:04.996759713Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"6b9de16a-e421-4831-ac89-299c0a70225e","remote_addr":"[fd00:1122:3344:101::b]:51324","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":55578,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:15:04.997086119Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"04c8d0b9-0090-4ad3-955f-4c9363ecae69","remote_addr":"[fd00:1122:3344:101::b]:51324","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:15:05.213396588Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:40187"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:15:05.265897862Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"8cc17dcd-1f87-4ef7-b88f-1ff3e3f0d911","remote_addr":"[fd00:1122:3344:101::a]:40187","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":52423,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:15:05.266182098Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"04a7bc39-61c3-41fc-9864-fa5e8af7ec60","remote_addr":"[fd00:1122:3344:101::a]:40187","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:15:05.413660012Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:46318"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:15:05.463518301Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"83ac0e0f-eae9-4ee6-a4de-e2b035e96582","remote_addr":"[fd00:1122:3344:101::c]:46318","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49749,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:15:05.463823998Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"c22c5ef7-8f27-4333-9921-051363ad7429","remote_addr":"[fd00:1122:3344:101::c]:46318","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:15:06.215049956Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58475"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:15:06.215193369Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"b5822e60-0d93-42f1-85d2-35462e6cdce7","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58475","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":102,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:15:20.745996359Z","hostname":"buskin","pid":1590,"uri":"//83429885-0785-47bc-b249-4b860f5e232d","method":"GET","req_id":"4ec0dfdb-dec2-44ae-98be-fcdb4bcbfddb","remote_addr":"[fd00:1122:3344:101::d]:57308","local_addr":"[fd00:1122:3344:101::1]:48878","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":536,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:15:35.16116494Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:39907"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:15:35.213892978Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"c3fbf7c2-1ef6-4664-a9bc-5539d82c2e08","remote_addr":"[fd00:1122:3344:101::b]:39907","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":52633,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:15:35.214167964Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"41c43798-d63a-41a2-a3e9-b86182d5165f","remote_addr":"[fd00:1122:3344:101::b]:39907","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:15:35.399028328Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:33858"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:15:35.44950028Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"d5090452-5112-412a-97af-9e3128128c4d","remote_addr":"[fd00:1122:3344:101::a]:33858","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50368,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:15:35.449798606Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"79dc2d36-4d31-4e03-8e1e-484adcc3b1bf","remote_addr":"[fd00:1122:3344:101::a]:33858","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":115,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:15:35.600268412Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:60605"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:15:35.650659793Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"6b25c115-081d-4a03-9c8d-ae8eedc2b27b","remote_addr":"[fd00:1122:3344:101::c]:60605","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50272,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:15:35.650930318Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"e4765bf4-7a21-4c22-9003-aeb1de84e338","remote_addr":"[fd00:1122:3344:101::c]:60605","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:15:36.248519034Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42642"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:15:36.248636326Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"ccfd9427-3b87-42ed-9373-083be74c9c30","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42642","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":84,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:15:50.770682639Z","hostname":"buskin","pid":1590,"uri":"//83429885-0785-47bc-b249-4b860f5e232d","method":"GET","req_id":"476c85f9-5345-431e-9612-a7056a24225e","remote_addr":"[fd00:1122:3344:101::d]:57308","local_addr":"[fd00:1122:3344:101::1]:48878","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":646,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:16:04.850943034Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:60865"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:16:04.900597389Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"9228c8c1-763f-4c89-b6ec-2cfead236685","remote_addr":"[fd00:1122:3344:101::b]:60865","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49611,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:16:04.900899365Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"3f18b033-4752-4104-912e-bdd4771841a0","remote_addr":"[fd00:1122:3344:101::b]:60865","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":116,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:16:05.168677836Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:38184"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:16:05.221093919Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"cc015a05-0eb7-4e0b-854f-2bcc785cb79a","remote_addr":"[fd00:1122:3344:101::a]:38184","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":52295,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:16:05.221349724Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"34ba984d-b68d-43e5-ac53-629feb0ada7d","remote_addr":"[fd00:1122:3344:101::a]:38184","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:16:05.31899458Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:37233"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:16:05.368246656Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"d6ab68a9-19ec-4e74-8223-7f6d9540ac5d","remote_addr":"[fd00:1122:3344:101::c]:37233","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49146,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:16:05.368510432Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"5b0634e5-bfb0-451c-9d4d-9adb17fc7c49","remote_addr":"[fd00:1122:3344:101::c]:37233","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":106,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:16:06.280213045Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35345"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:16:06.281530532Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"f47fffdc-a0e9-4eb3-8428-8f8169b08a50","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35345","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":717,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:16:20.840688439Z","hostname":"buskin","pid":1590,"uri":"//83429885-0785-47bc-b249-4b860f5e232d","method":"GET","req_id":"f5c00f19-2b0e-4a26-b12c-89c7c509e270","remote_addr":"[fd00:1122:3344:101::d]:57308","local_addr":"[fd00:1122:3344:101::1]:48878","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":809,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:16:35.27151299Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:39383"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:16:35.323852721Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"843a0839-23a0-403f-bd48-92e8b80319ad","remote_addr":"[fd00:1122:3344:101::b]:39383","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":52255,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:16:35.324104016Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"9b62fcab-6b33-460e-b85d-e2171a475a86","remote_addr":"[fd00:1122:3344:101::b]:39383","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:16:35.374356763Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:50327"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:16:35.419393892Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"63222c46-7b24-44d6-a7af-c8affe6694ab","remote_addr":"[fd00:1122:3344:101::a]:50327","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44942,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:16:35.420286611Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"170168a9-6d0e-45d4-82ff-53300f40ff39","remote_addr":"[fd00:1122:3344:101::a]:50327","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":737,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:16:35.534832468Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:56307"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:16:35.581836478Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"a8346cdb-763a-487d-829a-9277034bf5d3","remote_addr":"[fd00:1122:3344:101::c]:56307","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46904,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:16:35.582081963Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"b7cb9914-a64f-45c8-8b0c-a32cbfdead45","remote_addr":"[fd00:1122:3344:101::c]:56307","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:16:36.316352518Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:16:36.316478841Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"2684a224-90ae-417c-8eb6-63a405d87ff2","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52198","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":120,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:16:50.840577996Z","hostname":"buskin","pid":1590,"uri":"//83429885-0785-47bc-b249-4b860f5e232d","method":"GET","req_id":"107dc082-4070-4b03-b7a2-9b6ccab98c61","remote_addr":"[fd00:1122:3344:101::d]:57308","local_addr":"[fd00:1122:3344:101::1]:48878","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":567,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:17:04.90832885Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:41299"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:17:04.957933304Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"a74b128b-8d95-44d5-a072-888b62d788cc","remote_addr":"[fd00:1122:3344:101::b]:41299","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49547,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:17:04.958199709Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"2b05074c-e999-4505-ad47-d1d5350c0c5a","remote_addr":"[fd00:1122:3344:101::b]:41299","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:17:05.247759855Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:44822"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:17:05.298813119Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"e88f6026-30e1-4b8e-a52d-d5638d0c7829","remote_addr":"[fd00:1122:3344:101::a]:44822","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50943,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:17:05.299081335Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"86760610-56b0-46a3-b079-d1d16c656711","remote_addr":"[fd00:1122:3344:101::a]:44822","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:17:05.329389336Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:33412"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:17:05.374250461Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"ff82434c-99a6-434d-9407-cafa59282bc4","remote_addr":"[fd00:1122:3344:101::c]:33412","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44765,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:17:05.374511217Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"8355843f-f9d8-4781-b981-b0cb3adc81f2","remote_addr":"[fd00:1122:3344:101::c]:33412","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:17:06.352041252Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49191"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:17:06.352190705Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"6d5525d1-096b-4f3a-9ce3-5ea380799139","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49191","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":100,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:17:20.841474407Z","hostname":"buskin","pid":1590,"uri":"//83429885-0785-47bc-b249-4b860f5e232d","method":"GET","req_id":"f378d3f8-a569-40d1-a535-bda87a25f101","remote_addr":"[fd00:1122:3344:101::d]:57308","local_addr":"[fd00:1122:3344:101::1]:48878","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":940,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:17:35.263523147Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:54274"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:17:35.315155603Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"43379587-022e-426a-81ef-e9c5f913a4dd","remote_addr":"[fd00:1122:3344:101::b]:54274","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":51588,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:17:35.316427139Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"ebeaabe4-99fc-4b3a-b842-3c49c5566ee7","remote_addr":"[fd00:1122:3344:101::b]:54274","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":998,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:17:35.578637685Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:34258"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:17:35.6268242Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"57384e2b-c3b8-4124-a954-22bd1f59e61d","remote_addr":"[fd00:1122:3344:101::a]:34258","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48147,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:17:35.627109326Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"ac99228f-45d1-4f1d-9dca-a40a5f18fe32","remote_addr":"[fd00:1122:3344:101::a]:34258","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":113,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:17:35.656446757Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:57207"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:17:35.702897375Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"733fdb34-1626-4234-989e-889b61385432","remote_addr":"[fd00:1122:3344:101::c]:57207","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46344,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:17:35.703810454Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"d29bc141-3f0c-4589-97d2-e0d5458ab640","remote_addr":"[fd00:1122:3344:101::c]:57207","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":725,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:17:36.38486825Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56959"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:17:36.384960532Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"af36060e-ac33-4d10-b55c-65d0c02d96a3","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56959","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":74,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:17:50.842103504Z","hostname":"buskin","pid":1590,"uri":"//83429885-0785-47bc-b249-4b860f5e232d","method":"GET","req_id":"d4287624-01bb-4edc-a6cb-5e1a45c3e26b","remote_addr":"[fd00:1122:3344:101::d]:57308","local_addr":"[fd00:1122:3344:101::1]:48878","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1093,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:04.20321989Z","hostname":"buskin","pid":1590,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:300"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:04.204212351Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:04.251541448Z","hostname":"buskin","pid":1590,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.99999811s","next_cleanup":"Instant { tv_sec: 2187, tv_nsec: 348055829 }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:05.015802258Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:44015"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:05.067846543Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"5d5b6f31-b753-44a4-a52f-c9f1a7275744","remote_addr":"[fd00:1122:3344:101::b]:44015","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":51969,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:05.068103028Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"4bca64e5-5a47-45e0-aece-5e549670de71","remote_addr":"[fd00:1122:3344:101::b]:44015","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":90,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:05.199527037Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:60600"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:05.251147743Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"400ae717-018d-4e38-b744-d16f7b368a48","remote_addr":"[fd00:1122:3344:101::a]:60600","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":51515,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:05.251432289Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"047ae670-186c-4cbe-bccc-6b215ca2783e","remote_addr":"[fd00:1122:3344:101::a]:60600","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:05.4448136Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:35425"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:05.495503726Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"19cdb73f-6783-4fd6-a56f-d8c3b3aee01b","remote_addr":"[fd00:1122:3344:101::c]:35425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50594,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:05.495766252Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"78f5460d-3a98-4745-8864-efc8fcdcb953","remote_addr":"[fd00:1122:3344:101::c]:35425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:05.888470288Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:57674"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:05.903756346Z","hostname":"buskin","pid":1590,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:521","metadata":"InstanceMetadata { silo_id: a8dc0d5a-b723-4654-86e8-4938e63ef836, project_id: b6e29fef-6f2d-4d3a-9015-7d2a595ab8a8 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-05-01T05:18:05.411934Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(58c3e9f9-122a-401c-a848-cd619ede25ec), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-01T05:18:05.436523807Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: f0f1c582-ccc8-4c4e-8a65-5bebea9c8cd1, kind: Instance { id: 44490038-e91e-43ae-92be-1d3ed4ca694f }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 252, 206, 217])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5279616), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f0f1c582-ccc8-4c4e-8a65-5bebea9c8cd1, kind: Instance { id: 44490038-e91e-43ae-92be-1d3ed4ca694f }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 252, 206, 217])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5279616), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f0f1c582-ccc8-4c4e-8a65-5bebea9c8cd1, kind: Instance { id: 44490038-e91e-43ae-92be-1d3ed4ca694f }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 252, 206, 217])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5279616), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(5279616))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f0f1c582-ccc8-4c4e-8a65-5bebea9c8cd1, kind: Instance { id: 44490038-e91e-43ae-92be-1d3ed4ca694f }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 252, 206, 217])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5279616), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-db4c607a04b\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 9b489e9f-b7fc-41b5-8419-9e3a8abd00eb, read_only_parent: Some(Volume { block_size: 512, id: 65057449-e22c-44db-923e-967796b0557a, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: b6c4b164-d837-4d73-933e-5b0adbcf749c, key: Some(\"MWN/yI+MFqNMnt2CY83LueqIoPZKTBKC+MjMTsZ6DgM=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::19]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 9b489e9f-b7fc-41b5-8419-9e3a8abd00eb, key: Some(\"OLvf6soxHheXPTfkZanasEqLNCKNqv9sPGi9aJTvtek=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19000\", \"[fd00:1122:3344:101::14]:19000\", \"[fd00:1122:3344:101::12]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAC5QiqhWKFYAABCKqFYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAuUIqoVihWAAAQiqhWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI0NDQ5MDAzOC1lOTFlLTQzYWUtOTJiZS0xZDNlZDRjYTY5NGYiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJTDBuL0VuazJ2QlhVVDBWTWFGYlB5TERVVWp4djZ5aDVtZ0RqWTJib2ZXeiJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"58c3e9f9-122a-401c-a848-cd619ede25ec","instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:05.903881839Z","hostname":"buskin","pid":1590,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:555","instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:05.904850629Z","hostname":"buskin","pid":1590,"component":"InstanceManager","file":"sled-agent/src/instance.rs:938","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: f0f1c582-ccc8-4c4e-8a65-5bebea9c8cd1, kind: Instance { id: 44490038-e91e-43ae-92be-1d3ed4ca694f }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 252, 206, 217])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5279616), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f0f1c582-ccc8-4c4e-8a65-5bebea9c8cd1, kind: Instance { id: 44490038-e91e-43ae-92be-1d3ed4ca694f }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 252, 206, 217])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5279616), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f0f1c582-ccc8-4c4e-8a65-5bebea9c8cd1, kind: Instance { id: 44490038-e91e-43ae-92be-1d3ed4ca694f }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 252, 206, 217])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5279616), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(5279616))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f0f1c582-ccc8-4c4e-8a65-5bebea9c8cd1, kind: Instance { id: 44490038-e91e-43ae-92be-1d3ed4ca694f }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 252, 206, 217])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5279616), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-db4c607a04b\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 9b489e9f-b7fc-41b5-8419-9e3a8abd00eb, read_only_parent: Some(Volume { block_size: 512, id: 65057449-e22c-44db-923e-967796b0557a, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: b6c4b164-d837-4d73-933e-5b0adbcf749c, key: Some(\"MWN/yI+MFqNMnt2CY83LueqIoPZKTBKC+MjMTsZ6DgM=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::19]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 9b489e9f-b7fc-41b5-8419-9e3a8abd00eb, key: Some(\"OLvf6soxHheXPTfkZanasEqLNCKNqv9sPGi9aJTvtek=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19000\", \"[fd00:1122:3344:101::14]:19000\", \"[fd00:1122:3344:101::12]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAC5QiqhWKFYAABCKqFYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAuUIqoVihWAAAQiqhWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI0NDQ5MDAzOC1lOTFlLTQzYWUtOTJiZS0xZDNlZDRjYTY5NGYiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJTDBuL0VuazJ2QlhVVDBWTWFGYlB5TERVVWp4djZ5aDVtZ0RqWTJib2ZXeiJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(58c3e9f9-122a-401c-a848-cd619ede25ec), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-01T05:18:05.436523807Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-05-01T05:18:05.411934Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"58c3e9f9-122a-401c-a848-cd619ede25ec","instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:05.907105966Z","hostname":"buskin","pid":1590,"uri":"/instances/44490038-e91e-43ae-92be-1d3ed4ca694f","method":"PUT","req_id":"90d7f2e3-083c-46ab-982a-43277cac4586","remote_addr":"[fd00:1122:3344:101::a]:57674","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":18546,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:06.000157016Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:62745"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:06.00230024Z","hostname":"buskin","pid":1590,"uri":"/instances/44490038-e91e-43ae-92be-1d3ed4ca694f/state","method":"PUT","req_id":"4cbaa355-9a2c-4fb9-b41c-4bcdf01d6fff","remote_addr":"[fd00:1122:3344:101::a]:62745","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":2087,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:06.023494352Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 252, 206, 217]), slot: 0, vni: Vni { inner: 5279616 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:06.024158056Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:06.060064654Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:06.083433641Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:06.419401584Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38032"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:06.419556347Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"f962ae7a-a63b-4184-8c68-6169c4d8a6be","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38032","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":129,"response_code":"200"} {"msg":"Profile for oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:07.975508899Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:07.975820316Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec","instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec\") failed: Property not found. retry in 35.453832ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:18:08.185740071Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec","instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec\") failed: Property not found. retry in 54.032018ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:18:08.23029558Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec","instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec\") failed: Property not found. retry in 107.773909ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:18:08.299225277Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec","instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec\") failed: Property not found. retry in 581.001424ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:18:08.438497109Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec","instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec\") failed: Property not found. retry in 775.971156ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:18:09.032070342Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec","instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec\") failed: Property not found. retry in 1.159739224s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:18:09.820424054Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec","instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec\") failed: Property not found. retry in 1.018228953s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:18:10.991418492Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec","instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec\") failed: Property not found. retry in 682.902217ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:18:12.020211606Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec","instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec\") failed: Property not found. retry in 557.631476ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:18:12.714027648Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec","instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:13.301136085Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"sled-agent/src/instance.rs:1423"} {"msg":"wait for service Some(\"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec\") failed: Property not found. retry in 39.506376ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:18:13.310685815Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec\") failed: Property not found. retry in 69.130413ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:18:13.359345089Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec\") failed: Property not found. retry in 104.698894ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:18:13.438154121Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec\") failed: Property not found. retry in 351.463276ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:18:13.552688179Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:13.913693884Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"sled-agent/src/instance.rs:1432"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:13.948873277Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"sled-agent/src/instance.rs:1446"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAC5QiqhWKFYAABCKqFYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAuUIqoVihWAAAQiqhWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI0NDQ5MDAzOC1lOTFlLTQzYWUtOTJiZS0xZDNlZDRjYTY5NGYiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJTDBuL0VuazJ2QlhVVDBWTWFGYlB5TERVVWp4djZ5aDVtZ0RqWTJib2ZXeiJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-db4c607a04b\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 9b489e9f-b7fc-41b5-8419-9e3a8abd00eb, read_only_parent: Some(Volume { block_size: 512, id: 65057449-e22c-44db-923e-967796b0557a, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: b6c4b164-d837-4d73-933e-5b0adbcf749c, key: Some(\"MWN/yI+MFqNMnt2CY83LueqIoPZKTBKC+MjMTsZ6DgM=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::19]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 9b489e9f-b7fc-41b5-8419-9e3a8abd00eb, key: Some(\"OLvf6soxHheXPTfkZanasEqLNCKNqv9sPGi9aJTvtek=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19000\", \"[fd00:1122:3344:101::14]:19000\", \"[fd00:1122:3344:101::12]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 44490038-e91e-43ae-92be-1d3ed4ca694f, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: b6e29fef-6f2d-4d3a-9015-7d2a595ab8a8, silo_id: a8dc0d5a-b723-4654-86e8-4938e63ef836 }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:13.95379408Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"sled-agent/src/instance.rs:685"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:14.149409057Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"sled-agent/src/instance.rs:687"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-05-01T05:18:14.150647003Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:14.151331327Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:14.151346298Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-05-01T05:18:14.150647003Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(58c3e9f9-122a-401c-a848-cd619ede25ec), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-01T05:18:05.436523807Z }","propolis_id":"58c3e9f9-122a-401c-a848-cd619ede25ec"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:14.151364168Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(58c3e9f9-122a-401c-a848-cd619ede25ec), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-01T05:18:05.436523807Z }, propolis_id: 58c3e9f9-122a-401c-a848-cd619ede25ec, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-05-01T05:18:14.150647003Z } }","instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-05-01T05:18:14.255908037Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:14.255961468Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:14.25605933Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-05-01T05:18:14.255908037Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(58c3e9f9-122a-401c-a848-cd619ede25ec), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-01T05:18:05.436523807Z }","propolis_id":"58c3e9f9-122a-401c-a848-cd619ede25ec"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:14.25608611Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(58c3e9f9-122a-401c-a848-cd619ede25ec), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-01T05:18:05.436523807Z }, propolis_id: 58c3e9f9-122a-401c-a848-cd619ede25ec, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-05-01T05:18:14.255908037Z } }","instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:20.842056347Z","hostname":"buskin","pid":1590,"uri":"//83429885-0785-47bc-b249-4b860f5e232d","method":"GET","req_id":"e7239894-b876-4972-8476-7ae2a6eca25b","remote_addr":"[fd00:1122:3344:101::d]:57308","local_addr":"[fd00:1122:3344:101::1]:48878","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"83429885-0785-47bc-b249-4b860f5e232d","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":697,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:22.676999125Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:44323"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:22.678295401Z","hostname":"buskin","pid":1590,"uri":"/instances/44490038-e91e-43ae-92be-1d3ed4ca694f/state","method":"PUT","req_id":"83bff0d4-eca5-4d5d-be90-a86b6009d4cb","remote_addr":"[fd00:1122:3344:101::a]:44323","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1228,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-05-01T05:18:22.678423884Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:22.678450455Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:22.678469865Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-05-01T05:18:22.678423884Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(58c3e9f9-122a-401c-a848-cd619ede25ec), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-01T05:18:05.436523807Z }","propolis_id":"58c3e9f9-122a-401c-a848-cd619ede25ec"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:22.678489056Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(58c3e9f9-122a-401c-a848-cd619ede25ec), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-01T05:18:05.436523807Z }, propolis_id: 58c3e9f9-122a-401c-a848-cd619ede25ec, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-05-01T05:18:22.678423884Z } }","instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-05-01T05:18:22.731801227Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:22.731828567Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:22.731854758Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-05-01T05:18:22.731801227Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-01T05:18:22.731801227Z }","propolis_id":"58c3e9f9-122a-401c-a848-cd619ede25ec"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:22.731881769Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:22.731887509Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:22.732077233Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:22.732300507Z","hostname":"buskin","pid":1590,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:437","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/07066414-66e9-4cbf-a342-0d2cd3c1d2ca/debug/bundle/zone\", \"/pool/int/57d48f42-fa69-4c1b-bfa2-9b1bf3d7ab45/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/4ab38590-bc9f-4317-9a2d-902570f44129/crypt/debug\", \"/pool/ext/f2ad007c-d0d5-499a-a92a-f0e5d6ac476c/crypt/debug\", \"/pool/ext/eeebb608-8f09-47dd-a873-6005f1a33225/crypt/debug\", \"/pool/ext/4917eed2-7e51-43de-bbf3-90e6e217a77b/crypt/debug\", \"/pool/ext/137f7f84-2894-49f9-be7d-163348c068b9/crypt/debug\", \"/pool/ext/7673312c-f216-4503-9437-30203b65ec1d/crypt/debug\", \"/pool/ext/4237ec86-747e-4b2e-a81d-30673b143f9c/crypt/debug\", \"/pool/ext/0a8cb798-6ba7-43a1-aa2f-7aba85873942/crypt/debug\", \"/pool/ext/e211498b-7438-4a66-89c4-ebf7a721e12d/crypt/debug\"] }","zone_name":"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:23.347063471Z","hostname":"buskin","pid":1590,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1230","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec\", bundle_id: 4dddd6f8-2921-4ffb-9202-a866ba861e03 }, time_created: 2024-05-01T05:18:22.732627054Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_58c3e9f9-122a-401c-a848-cd619ede25ec","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:18:23.347162193Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:25.113036241Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:25.140714508Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-01T05:18:22.731801227Z }, propolis_id: 58c3e9f9-122a-401c-a848-cd619ede25ec, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-05-01T05:18:25.140641417Z } }","instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:25.241818215Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:63447"} {"msg":"unset_virtual_nic_host unimplmented","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T05:18:25.2420588Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:25.24207798Z","hostname":"buskin","pid":1590,"uri":"/v2p/f0f1c582-ccc8-4c4e-8a65-5bebea9c8cd1","method":"DELETE","req_id":"834c9af5-2230-4a8f-84fe-d318f5f94e56","remote_addr":"[fd00:1122:3344:101::b]:63447","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":149,"response_code":"204"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:25.556856942Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-01T05:18:22.731801227Z }, propolis_id: 58c3e9f9-122a-401c-a848-cd619ede25ec, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-05-01T05:18:25.140641417Z } }","instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T05:18:25.560710712Z","hostname":"buskin","pid":1590,"instance_id":"44490038-e91e-43ae-92be-1d3ed4ca694f","component":"InstanceManager","file":"sled-agent/src/instance.rs:723"}