[ Jun 5 19:48:20 Enabled. ] [ Jun 5 19:48:20 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Jun 5 19:48:20 Method "start" exited with status 0. ] [ Jun 5 19:48:20 Rereading configuration. ] [ Jun 5 19:48:20 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:20.136707186Z","hostname":"buskin","pid":1555,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:20.136869179Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:20.190040415Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:20.190086896Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:20.303864572Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:20.312735391Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:20.320935975Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:20.921857572Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:9fb2::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.070147902Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:119"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.070194473Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:131"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.070305376Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.070428599Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:162","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.070441469Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:506"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.07047781Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:625"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.073255866Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:21.073263006Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:674"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.073281986Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.073290046Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:733"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.073305407Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.073343228Z","hostname":"buskin","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.073395019Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.073412059Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.07342052Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.084676015Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.084713266Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"ca6bd534-e33c-4c83-bc67-7b70494fb021 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.183366195Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: ca6bd534-e33c-4c83-bc67-7b70494fb021 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.386387678Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.386418629Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.386425969Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.38644151Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.38644823Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.38645289Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.38647137Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.395802331Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.395830541Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.4923625Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699131082Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699170513Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: ca6bd534-e33c-4c83-bc67-7b70494fb021 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7 (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699192583Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699210683Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699218584Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699260865Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699296705Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_1.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699320266Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699324946Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: ca6bd534-e33c-4c83-bc67-7b70494fb021 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699338397Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699343487Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699348797Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699356647Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699364747Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: ca6bd534-e33c-4c83-bc67-7b70494fb021 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699386688Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699392248Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699396608Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699401918Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699406588Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699411878Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: ca6bd534-e33c-4c83-bc67-7b70494fb021 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699428559Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699434209Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699439219Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699445829Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699450959Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_4.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699455789Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699460209Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: ca6bd534-e33c-4c83-bc67-7b70494fb021 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.69948707Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.69949244Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.69949769Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699502801Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_5.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.69950809Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699512801Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: ca6bd534-e33c-4c83-bc67-7b70494fb021 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699531441Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699538501Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699544651Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699548651Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_6.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699552892Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: ca6bd534-e33c-4c83-bc67-7b70494fb021 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699580112Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699585622Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699590823Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699594852Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699599523Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_7.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699607203Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: ca6bd534-e33c-4c83-bc67-7b70494fb021 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699622923Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699628693Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699632664Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699636624Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699639814Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699650934Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699654564Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: ca6bd534-e33c-4c83-bc67-7b70494fb021 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699673434Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699677914Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699682275Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699687805Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:91"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699696705Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:93"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699700025Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: ca6bd534-e33c-4c83-bc67-7b70494fb021 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699726516Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699731106Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699734726Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699738226Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"No ledger in /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699741846Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699767127Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699782587Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699789717Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699793857Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699797617Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699820068Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.699825588Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.709983408Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:711"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.710010398Z","hostname":"buskin","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999936928s","next_cleanup":"Instant { tv_sec: 1333, tv_nsec: 56873152 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.71009952Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3645"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.710105411Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.710168332Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.710180853Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.713387768Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.71345499Z","hostname":"buskin","pid":1555,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV1 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV1 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n },\n ],\n addresses: [\n V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.713658645Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1004"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.713664955Z","hostname":"buskin","pid":1555,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.713714406Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.713755817Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.713768927Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.713778567Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1064"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.713804798Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.713817908Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.713834949Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.713839239Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1105"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.713842809Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.713846389Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.713857789Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.71387199Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.714244858Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.714387272Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.714394732Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1138"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.714424823Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.714429893Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.714572046Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.714707379Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.71473915Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 8fcf3569-af9e-4967-b82f-54e3a2e91f58, rack_id: 2c4cdae1-0bef-46b7-80a3-3be5f1adb047, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.714839912Z","hostname":"buskin","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:61404","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 8fcf3569-af9e-4967-b82f-54e3a2e91f58, rack_id: 2c4cdae1-0bef-46b7-80a3-3be5f1adb047, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.714990026Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.714996566Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.715028547Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.715034007Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:530"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.715043407Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/server.rs:44"} {"msg":"No ledger in /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.715050528Z","hostname":"buskin","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.715055468Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.715062658Z","hostname":"buskin","pid":1555,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"No ledger in /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.715067798Z","hostname":"buskin","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.715075058Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:519"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.715079828Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:547"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.717624268Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3877"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.724843129Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.734601949Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.745695861Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.756172048Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.764782582Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.773071327Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.781332002Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.789538796Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.797642357Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.797689679Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:558"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.797694328Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.797698709Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:370"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.79775809Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.79776463Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:449","result":"DisksManagementResult { status: [] }"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.817525477Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.833993745Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.860550283Z","hostname":"buskin","pid":1555,"sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:353"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.860600734Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.860606504Z","hostname":"buskin","pid":1555,"sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:365"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.860610134Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 60.669442ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.863674686Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0ca65427-cf8e-43de-9ebf-416ead675e97\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:48:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0ca65427-cf8e-43de-9ebf-416ead675e97\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_ca6bd534-e33c-4c83-bc67-7b70494fb021/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.873391636Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_ca6bd534-e33c-4c83-bc67-7b70494fb021/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.897108986Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_ca6bd534-e33c-4c83-bc67-7b70494fb021/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.897157197Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.902475652Z","hostname":"buskin","pid":1555,"sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:380"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.902491023Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_ca6bd534-e33c-4c83-bc67-7b70494fb021/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.902497153Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.924583484Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 89.078789ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.925021825Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1e7bad12-10b3-44dd-b55b-6ad01b0e51c6\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:48:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1e7bad12-10b3-44dd-b55b-6ad01b0e51c6\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:21.973173181Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.015271665Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 73.759865ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.015881799Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a106532e-b518-4213-b6c2-256a59a861d4\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:48:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a106532e-b518-4213-b6c2-256a59a861d4\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.090936061Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 97.41581ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.091433293Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ca846de4-e016-4c69-bf78-40f085291eb1\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:48:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ca846de4-e016-4c69-bf78-40f085291eb1\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.190133643Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 206.727657ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.190537373Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6b73c33c-bbe7-48d7-981c-b5edea7942b6\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:48:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6b73c33c-bbe7-48d7-981c-b5edea7942b6\" }"} {"msg":"Mounting oxi_ca6bd534-e33c-4c83-bc67-7b70494fb021/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.192760265Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.205543207Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.205624579Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.205642539Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.20567367Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.397696343Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 324.463795ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.39839901Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6b01bb91-4a80-4235-95e1-9e9499382077\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:48:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6b01bb91-4a80-4235-95e1-9e9499382077\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.724072849Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 249.352556ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.724718374Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d2e09d2d-7d6e-41e0-b5f2-a9becc95af2c\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:48:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d2e09d2d-7d6e-41e0-b5f2-a9becc95af2c\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.927889061Z","hostname":"buskin","pid":1555,"sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.928467234Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:59238","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.928492105Z","hostname":"buskin","pid":1555,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.928557946Z","hostname":"buskin","pid":1555,"file":"oximeter/producer/src/lib.rs:276","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"oximeter/producer/src/lib.rs:276","interval":"30s","address":"[fd00:1122:3344:101::1]:59238","producer_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:22.929673683Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"261.103128ms"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.974639504Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.981235569Z","hostname":"buskin","pid":1555,"component":"VmmReservoirManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.98124819Z","hostname":"buskin","pid":1555,"component":"VmmReservoirManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.981413404Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:993","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.981442834Z","hostname":"buskin","pid":1555,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.982695434Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.982731525Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.982767496Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.982799566Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.983062232Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.983271408Z","hostname":"buskin","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.983299828Z","hostname":"buskin","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.1.100, vlan_id: None }], addresses: [V4(Ipv4Net { addr: 10.151.1.174, width: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.983310578Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3645"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:22.983316338Z","hostname":"buskin","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:61404","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.190047239Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.190114041Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.190172382Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.190248554Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.190296285Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.190305405Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.190320265Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:23.192250571Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"695.042422ms"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8fcf3569-af9e-4967-b82f-54e3a2e91f58): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:23.194796701Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.222633788Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:54182"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.222876234Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.223001527Z","hostname":"buskin","pid":1555,"uri":"/inventory","method":"GET","req_id":"229d0f62-918b-44c6-8455-66a2dc20dd26","remote_addr":"[fd00:1122:3344:101::1]:54182","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":211,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.25276488Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:65397"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.252875562Z","hostname":"buskin","pid":1555,"uri":"/sled-role","method":"GET","req_id":"d78c130d-067d-40b7-bdc2-b0ccbb40bd7c","remote_addr":"[fd00:1122:3344:101::1]:65397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":52,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.25446844Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.254502951Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.254921751Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.255159836Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:815"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: 2f01adc7-e4c9-43d4-9d69-a0426e674a5c, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, pool_id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool) }, OmicronPhysicalDiskConfig { id: e5bd0243-4275-48d2-a538-57b2c5682ca1, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, pool_id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool) }, OmicronPhysicalDiskConfig { id: 9f4032e3-8404-4dfb-b20c-970d21a60e72, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, pool_id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool) }, OmicronPhysicalDiskConfig { id: 714d61a3-9020-4bf1-9ab9-2c9714819c87, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, pool_id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool) }, OmicronPhysicalDiskConfig { id: e7b373ee-431a-43b7-b76f-ac5c8b8a0af3, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, pool_id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool) }, OmicronPhysicalDiskConfig { id: 4e9807d4-2abb-4512-937f-bb9d98ac46bf, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, pool_id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool) }, OmicronPhysicalDiskConfig { id: 85f65b50-b24e-4d73-8414-21a3bf350cf5, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, pool_id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool) }, OmicronPhysicalDiskConfig { id: 6016e12f-99c7-42a4-b015-e41fe9f141ef, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, pool_id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool) }, OmicronPhysicalDiskConfig { id: 83a13742-4aaf-4e8b-9215-c5d2edf327a8, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, pool_id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.286424285Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:348"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.286558618Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:63240"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 2f01adc7-e4c9-43d4-9d69-a0426e674a5c, pool_id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: e5bd0243-4275-48d2-a538-57b2c5682ca1, pool_id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 9f4032e3-8404-4dfb-b20c-970d21a60e72, pool_id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 714d61a3-9020-4bf1-9ab9-2c9714819c87, pool_id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: e7b373ee-431a-43b7-b76f-ac5c8b8a0af3, pool_id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 4e9807d4-2abb-4512-937f-bb9d98ac46bf, pool_id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 85f65b50-b24e-4d73-8414-21a3bf350cf5, pool_id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 6016e12f-99c7-42a4-b015-e41fe9f141ef, pool_id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 83a13742-4aaf-4e8b-9215-c5d2edf327a8, pool_id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.286914966Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.286921026Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.286928276Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.286933406Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:681"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.286950997Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.286956327Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.286962177Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.286966557Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.387805768Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:23.387843279Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"906c9449-467f-496f-ae4c-f2197a847844 (zpool)"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:24.012538056Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"693.296371ms"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fcf3569-af9e-4967-b82f-54e3a2e91f58): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-05T19:48:24.013313734Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:24.411210838Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:24.591502345Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:24.591695469Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:24.591705629Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:24.5917423Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:24.591755101Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:24.591919945Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:24.591926455Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:24.796295299Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.487797181s"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8fcf3569-af9e-4967-b82f-54e3a2e91f58): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:24.993403623Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:25.178693067Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:25.368515878Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fcf3569-af9e-4967-b82f-54e3a2e91f58): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-05T19:48:25.560001589Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Automatically destroying dataset: oxp_906c9449-467f-496f-ae4c-f2197a847844/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:25.781707143Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 51.604552ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:25.965705557Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 140.890834ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:26.516861699Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:26.519133282Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.245263074s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fcf3569-af9e-4967-b82f-54e3a2e91f58): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-05T19:48:26.924136954Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 246.75197ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:27.134268414Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8fcf3569-af9e-4967-b82f-54e3a2e91f58): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:27.489802428Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 573.748199ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:27.656947244Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fcf3569-af9e-4967-b82f-54e3a2e91f58): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-05T19:48:28.038903711Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_906c9449-467f-496f-ae4c-f2197a847844","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:28.587933612Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 932.077151ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:28.818422954Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:28.820104974Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"5.110017249s"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:28.825957642Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:28.825969302Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:28.825983883Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:28.825988563Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:28.825995823Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:29.062420344Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:29.062463695Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"d69c4ddc-54d3-4718-a115-7582c8252249 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:30.256137566Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:30.488994683Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:30.489047294Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:30.489052874Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:30.489059034Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:30.489087525Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:30.4892826Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:30.48929332Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8fcf3569-af9e-4967-b82f-54e3a2e91f58): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:30.760123944Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fcf3569-af9e-4967-b82f-54e3a2e91f58): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-05T19:48:30.76037782Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:31.270542373Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_d69c4ddc-54d3-4718-a115-7582c8252249/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:32.004454609Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fcf3569-af9e-4967-b82f-54e3a2e91f58): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-05T19:48:32.760702303Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8fcf3569-af9e-4967-b82f-54e3a2e91f58): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:32.994194205Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.453684193Z","hostname":"buskin","pid":1555,"component":"VmmReservoirManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"Looking for unencrypted datasets in oxp_d69c4ddc-54d3-4718-a115-7582c8252249","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.539439127Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.548206074Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.548274536Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.548283716Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.548290207Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.548295277Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.558009516Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.558043137Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.651153915Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.659363369Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.65943104Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.659447081Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.659464131Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.659475252Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.659583884Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.659593974Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.695015131Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_8e7826e7-938b-4b78-bd9b-c82573a27350/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.703082221Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fcf3569-af9e-4967-b82f-54e3a2e91f58): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-05T19:48:33.763407825Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_8e7826e7-938b-4b78-bd9b-c82573a27350","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.826871613Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.835052937Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.835099118Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.835115118Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.835121118Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.835126158Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.84411558Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.844143541Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"87fce040-2c58-4185-8fd5-7dfb9378086a (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.934619527Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.943319533Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.943358334Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.943367904Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.943487886Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.943580079Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.943766353Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.943785524Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:33.947996883Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"13.256530769s"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.980717865Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_87fce040-2c58-4185-8fd5-7dfb9378086a/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:33.988633352Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_87fce040-2c58-4185-8fd5-7dfb9378086a","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.107026077Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.114172466Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.114211517Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.114218637Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.114225527Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.114231897Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.1240939Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.124130391Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.219901802Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.227553743Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.227591093Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.227601894Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.227626524Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.227637105Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.227820439Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.227828409Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.263146813Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_906c0a61-9cfc-4b51-be6b-6990f707b063/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.271131721Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_906c0a61-9cfc-4b51-be6b-6990f707b063","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.394168206Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.40195213Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.401984411Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.401990351Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.401994741Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.402017842Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.411764572Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.411806133Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.503539608Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.510592045Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.510651216Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.510660756Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.510692867Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.510701577Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.510836351Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.510841771Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8fcf3569-af9e-4967-b82f-54e3a2e91f58): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:34.514527358Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.545764395Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_ca57da68-282d-4e81-9a3e-bfb822551ad8/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.552860933Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_ca57da68-282d-4e81-9a3e-bfb822551ad8","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.679882021Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.687696346Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.687754507Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.687765097Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.687774878Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.687783328Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.698179313Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.698216594Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.793134515Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.800730744Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.800769205Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.800797726Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.800834947Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.800843037Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.801000151Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.801009411Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.83102639Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2331"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.839729285Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fcf3569-af9e-4967-b82f-54e3a2e91f58): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-05T19:48:34.843664578Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Automatically destroying dataset: oxp_df264fcd-b75d-4ee3-9b2e-e281836815f9/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.84840498Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_df264fcd-b75d-4ee3-9b2e-e281836815f9","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.96659061Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.97465297Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.974685331Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.974691181Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.974702481Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.974709351Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.983706783Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:34.983729184Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.076959855Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.084541884Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.084587375Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.084593875Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.084634866Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.084648007Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.084815811Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.084822881Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.120891632Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_9fef4b9e-a443-49fc-b47e-d8cdc87127ad/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.128733598Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_9fef4b9e-a443-49fc-b47e-d8cdc87127ad","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.251971867Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.25974804Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.259831073Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.259843893Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.259854693Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.259862063Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.269641014Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.269675965Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"c495d42b-321e-4420-8635-102c37b30fb2 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.361260417Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.369177234Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.369221545Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.369228495Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.369249786Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.369270996Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.36944619Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.36945831Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.406356382Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_c495d42b-321e-4420-8635-102c37b30fb2/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.414662578Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_c495d42b-321e-4420-8635-102c37b30fb2","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.537773404Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.545795814Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.545851135Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.545859515Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.546031229Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: ca6bd534-e33c-4c83-bc67-7b70494fb021 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.546100801Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.546110701Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Writing ledger to /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.546162042Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.54650695Z","hostname":"buskin","pid":1555,"uri":"/omicron-physical-disks","method":"PUT","req_id":"9d45a4c6-9494-40cf-8852-1726a9f03d98","remote_addr":"[fd00:1122:3344:101::1]:63240","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":12259811,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.578286751Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.578431554Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:49315"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.578705101Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.578732961Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.627858231Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.627903342Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.677703498Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.677762569Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.725447325Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.725515247Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.725528507Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.725575208Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.769001533Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.769050034Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.769055564Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_05f414eb-1b07-4b76-9d7c-27a7db52ad4b","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.821079973Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_46dff9fe-33a8-4272-a4b9-397eefbef96d","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.824831651Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_6d2b94d3-980a-43a1-8e8f-0973dee40a75","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.828611581Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_05f414eb-1b07-4b76-9d7c-27a7db52ad4b","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.841533386Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_46dff9fe-33a8-4272-a4b9-397eefbef96d","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.845903759Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fcf3569-af9e-4967-b82f-54e3a2e91f58): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-05T19:48:35.850021996Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Installing Omicron zone: oxz_internal_dns_6d2b94d3-980a-43a1-8e8f-0973dee40a75","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:35.850295982Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8fcf3569-af9e-4967-b82f-54e3a2e91f58): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:36.512911605Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fcf3569-af9e-4967-b82f-54e3a2e91f58): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-05T19:48:36.852039702Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Profile for oxz_internal_dns_6d2b94d3-980a-43a1-8e8f-0973dee40a75:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:37.366682931Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:37.366751973Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:37.3670478Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_6d2b94d3-980a-43a1-8e8f-0973dee40a75","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_internal_dns_05f414eb-1b07-4b76-9d7c-27a7db52ad4b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:37.851136478Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:38.338913673Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"Profile for oxz_internal_dns_46dff9fe-33a8-4272-a4b9-397eefbef96d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:38.339017786Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_internal_dns_6d2b94d3-980a-43a1-8e8f-0973dee40a75\") failed: Property not found. retry in 39.973005ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:38.346314468Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_6d2b94d3-980a-43a1-8e8f-0973dee40a75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:38.346352889Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:38.34640897Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_05f414eb-1b07-4b76-9d7c-27a7db52ad4b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:38.350957238Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_46dff9fe-33a8-4272-a4b9-397eefbef96d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_6d2b94d3-980a-43a1-8e8f-0973dee40a75\") failed: Property not found. retry in 100.002422ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:38.394955616Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_6d2b94d3-980a-43a1-8e8f-0973dee40a75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_46dff9fe-33a8-4272-a4b9-397eefbef96d\") failed: Property not found. retry in 45.516633ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:38.472821405Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_46dff9fe-33a8-4272-a4b9-397eefbef96d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_05f414eb-1b07-4b76-9d7c-27a7db52ad4b\") failed: Property not found. retry in 66.365924ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:38.478903749Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_05f414eb-1b07-4b76-9d7c-27a7db52ad4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6d2b94d3-980a-43a1-8e8f-0973dee40a75\") failed: Property not found. retry in 216.772643ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:38.503005607Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_6d2b94d3-980a-43a1-8e8f-0973dee40a75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_46dff9fe-33a8-4272-a4b9-397eefbef96d\") failed: Property not found. retry in 52.479593ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:38.526591284Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_46dff9fe-33a8-4272-a4b9-397eefbef96d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_05f414eb-1b07-4b76-9d7c-27a7db52ad4b\") failed: Property not found. retry in 128.446215ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:38.654570846Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_05f414eb-1b07-4b76-9d7c-27a7db52ad4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_46dff9fe-33a8-4272-a4b9-397eefbef96d\") failed: Property not found. retry in 282.895641ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:38.680417516Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_46dff9fe-33a8-4272-a4b9-397eefbef96d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6d2b94d3-980a-43a1-8e8f-0973dee40a75\") failed: Property not found. retry in 447.781488ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:38.739405748Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_6d2b94d3-980a-43a1-8e8f-0973dee40a75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_05f414eb-1b07-4b76-9d7c-27a7db52ad4b\") failed: Property not found. retry in 253.809389ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:38.800603653Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_05f414eb-1b07-4b76-9d7c-27a7db52ad4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_46dff9fe-33a8-4272-a4b9-397eefbef96d\") failed: Property not found. retry in 361.170618ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:38.97540655Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_46dff9fe-33a8-4272-a4b9-397eefbef96d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_05f414eb-1b07-4b76-9d7c-27a7db52ad4b\") failed: Property not found. retry in 370.216448ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:39.065302752Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_05f414eb-1b07-4b76-9d7c-27a7db52ad4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6d2b94d3-980a-43a1-8e8f-0973dee40a75\") failed: Property not found. retry in 541.353201ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:39.19608364Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_6d2b94d3-980a-43a1-8e8f-0973dee40a75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_46dff9fe-33a8-4272-a4b9-397eefbef96d\") failed: Property not found. retry in 408.727107ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:39.346230594Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_46dff9fe-33a8-4272-a4b9-397eefbef96d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_05f414eb-1b07-4b76-9d7c-27a7db52ad4b\") failed: Property not found. retry in 747.082428ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:39.445458407Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_05f414eb-1b07-4b76-9d7c-27a7db52ad4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6d2b94d3-980a-43a1-8e8f-0973dee40a75\") failed: Property not found. retry in 1.425319786s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:39.747271702Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_6d2b94d3-980a-43a1-8e8f-0973dee40a75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_46dff9fe-33a8-4272-a4b9-397eefbef96d\") failed: Property not found. retry in 1.026555301s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:39.767198593Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_46dff9fe-33a8-4272-a4b9-397eefbef96d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_05f414eb-1b07-4b76-9d7c-27a7db52ad4b\") failed: Property not found. retry in 702.824476ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:40.204269581Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_05f414eb-1b07-4b76-9d7c-27a7db52ad4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_46dff9fe-33a8-4272-a4b9-397eefbef96d\") failed: Property not found. retry in 1.012920929s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:40.802358151Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_46dff9fe-33a8-4272-a4b9-397eefbef96d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:9fb2::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:40.886166789Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2348"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:40.886178909Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"wait for service Some(\"oxz_internal_dns_05f414eb-1b07-4b76-9d7c-27a7db52ad4b\") failed: Property not found. retry in 1.452570165s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:40.915383999Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_05f414eb-1b07-4b76-9d7c-27a7db52ad4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6d2b94d3-980a-43a1-8e8f-0973dee40a75\") failed: Property not found. retry in 726.782312ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:41.182454194Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_6d2b94d3-980a-43a1-8e8f-0973dee40a75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe56:6a04","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:41.433686575Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2355"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:41.439685597Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2416"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:41.560811166Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2601"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:41.810846309Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2765"} {"msg":"wait for service Some(\"oxz_internal_dns_46dff9fe-33a8-4272-a4b9-397eefbef96d\") failed: Property not found. retry in 1.078709643s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:41.823619761Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_46dff9fe-33a8-4272-a4b9-397eefbef96d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6d2b94d3-980a-43a1-8e8f-0973dee40a75\") failed: Property not found. retry in 1.170164234s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:41.918070061Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_6d2b94d3-980a-43a1-8e8f-0973dee40a75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:42.002998535Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2480"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:42.304411111Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2515"} {"msg":"wait for service Some(\"oxz_internal_dns_05f414eb-1b07-4b76-9d7c-27a7db52ad4b\") failed: Property not found. retry in 764.100765ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:42.377202509Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_05f414eb-1b07-4b76-9d7c-27a7db52ad4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:42.526799301Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2815"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:42.707786554Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2842"} {"msg":"wait for service Some(\"oxz_internal_dns_46dff9fe-33a8-4272-a4b9-397eefbef96d\") failed: Property not found. retry in 572.292405ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:42.911298639Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_46dff9fe-33a8-4272-a4b9-397eefbef96d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:42.92914845Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2709"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:43.113661256Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2509"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:43.125118366Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3901","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:43.125174098Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3917"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:43.125184508Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"wait for service Some(\"oxz_internal_dns_05f414eb-1b07-4b76-9d7c-27a7db52ad4b\") failed: Property not found. retry in 967.354937ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:43.150750301Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_05f414eb-1b07-4b76-9d7c-27a7db52ad4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:43.680397385Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3925"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:43.6873789Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3981"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:43.82544803Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4043"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:43.926747771Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4024","rack_subnet":"fd00:1122:3344::"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:43.948595227Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4071"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.045834852Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4098"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.142230698Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_internal_dns_05f414eb-1b07-4b76-9d7c-27a7db52ad4b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.14229938Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_internal_dns_46dff9fe-33a8-4272-a4b9-397eefbef96d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.14230521Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_internal_dns_6d2b94d3-980a-43a1-8e8f-0973dee40a75"} {"msg":"Writing ledger to /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.142342671Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.14271881Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.142999386Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"ef45d2ad-6073-4f0c-b8a7-fe71299a979d","remote_addr":"[fd00:1122:3344:101::1]:49315","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8564495,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.144740447Z","hostname":"buskin","pid":1555,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.172512043Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:365"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.207050888Z","hostname":"buskin","pid":1555,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.216722477Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:413"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.255071502Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:651"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.255157564Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:689"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.255456821Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:692","uptime":"3"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.255468162Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:435","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.287430706Z","hostname":"buskin","pid":1555,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.328880515Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.328944996Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.328955276Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.328964747Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.328970607Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.328977567Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.328986157Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.328993017Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.328997317Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.329003198Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.329009658Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.329014958Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.329026808Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.329032438Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.329037658Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.329041968Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.329046099Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.329050169Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.329054329Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.329058449Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.329062729Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.329066939Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.329071139Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.348935208Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:585"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.348944618Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.381071437Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.381806104Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.381813254Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6fc5bc0c-95b6-40a3-a5a9-955d74c48961, kind: Service { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee }, name: Name(\"ntp-f73788bd-d6b7-4d17-b028-bd04b6dd3aee\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.410047261Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.410166464Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:36810"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.410321317Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.410326727Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.429953761Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.429987291Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.447895754Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.447929225Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.466083884Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.466116355Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.466291879Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.466378051Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.477585495Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.508918895Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.509577521Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.509582551Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.553008956Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 250, 116]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.553067037Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.553506598Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.553518578Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.553781134Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.55485733Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.577353831Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:44.595007658Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:45.209991836Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:45.210142859Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee\") failed: Property not found. retry in 63.988619ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:45.344132433Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee\") failed: Property not found. retry in 126.621696ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:45.42535111Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee\") failed: Property not found. retry in 234.91276ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:45.579738155Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee\") failed: Property not found. retry in 451.541348ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:45.824197876Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee\") failed: Property not found. retry in 507.425518ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:46.286525381Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee\") failed: Property not found. retry in 1.043736382s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:46.803768152Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:47.592675086Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46463"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:47.592781139Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"e3368da0-4dc7-4e36-b0dd-133567d12265","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46463","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee\") failed: Property not found. retry in 1.20221953s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:47.856684829Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee\") failed: Property not found. retry in 929.849469ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:49.068652521Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee\") failed: Property not found. retry in 985.830454ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:50.008239473Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:51.01285668Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee"} {"msg":"Writing ledger to /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:51.012915251Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:51.013249499Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:51.013517446Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"26370dc6-6abc-4708-b7ad-2ce2c1937ab1","remote_addr":"[fd00:1122:3344:101::1]:36810","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":6603306,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:51.015745358Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:626"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:51.045136732Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:51.045280905Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:32847"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:51.054151435Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3631"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:51.054309338Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"9cba766e-b28d-429d-a2f1-9110f2197a58","remote_addr":"[fd00:1122:3344:101::1]:32847","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8839,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:51.057128955Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:51.200168722Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:51.200277785Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:62460"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:55.209698069Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3631"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:55.209908164Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"88211708-ca49-4aad-90f8-aee6fe630a5b","remote_addr":"[fd00:1122:3344:101::1]:62460","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009529,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:55.213768105Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:55.376366594Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:55.376470126Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:40054"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:55.384946176Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3631"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:55.38511072Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"402b097e-db35-4099-be15-9e8d3a8a448e","remote_addr":"[fd00:1122:3344:101::1]:40054","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8549,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:55.38723037Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:55.518631432Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:55.518731475Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:64535"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_f73788bd-d6b7-4d17-b028-bd04b6dd3aee': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:59.528211571Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3631"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:59.528351284Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"3eca1195-54ff-4628-bf9a-f99daed88c00","remote_addr":"[fd00:1122:3344:101::1]:64535","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009535,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:59.53072797Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:59.781807898Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:59.781932531Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:43834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:59.790105584Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"a89ad10f-7528-40cc-8e40-3417f825e84c","remote_addr":"[fd00:1122:3344:101::1]:43834","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8099,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:48:59.791707191Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:48:59.791714982Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:00.094265194Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:00.094359696Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:41699"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:00.10256488Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"1020abd9-38fd-4fec-b91c-da1825961965","remote_addr":"[fd00:1122:3344:101::1]:41699","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8130,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:00.104884574Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:00.104890315Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:00.436997635Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:00.437098038Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:39951"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:00.444870981Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"7968b646-6d67-4057-9b49-e46ec1e1fe34","remote_addr":"[fd00:1122:3344:101::1]:39951","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":7692,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:00.447166545Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:00.447173745Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:00.652435161Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:00.652528613Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:46567"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:00.660262366Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"059258d9-87db-48f1-82d1-aed3a480b3fe","remote_addr":"[fd00:1122:3344:101::1]:46567","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":7646,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:00.662666703Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:00.662673063Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:00.884297195Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:00.884386737Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:49474"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:00.891532906Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"3bba1866-9308-4f6d-829d-a5460b8b1ab7","remote_addr":"[fd00:1122:3344:101::1]:49474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":7088,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:00.893534203Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:00.893539743Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.292808753Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.292904285Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:46816"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.312111198Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"2c60506e-e8df-4975-8d6c-d81a31e52f57","remote_addr":"[fd00:1122:3344:101::1]:46816","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":19117,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 396892370, ip_addr: 23.168.24.210, stratum: 3, ref_time: 1717616940.3024733, correction: -0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.314362201Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.314376142Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1217"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6fc5bc0c-95b6-40a3-a5a9-955d74c48961, kind: Service { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee }, name: Name(\"ntp-f73788bd-d6b7-4d17-b028-bd04b6dd3aee\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: b46b9748-75d3-4248-95bb-b19942b724b7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.344510603Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.344613225Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:57929"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.344767459Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.3447843Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.364497275Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.364562427Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.384733063Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.384764563Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.403746812Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.403797773Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.456179869Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.45621274Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.507926081Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.507963562Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.559096919Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.55913871Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.613619636Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.613657607Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.6641884Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.664231191Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.664463297Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.664577739Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.745351166Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.745396497Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.745418578Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.745424378Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.745430328Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.833852546Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.837607944Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_f6c2c6c4-927e-496d-873b-0e47ead3c40a","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.84167073Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_be258c33-8299-48b4-94e9-f6c15d8dd8a3","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.846106105Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_bb4b3cdb-0d2e-44b8-994b-0908726ddaeb","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.850852347Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.855439065Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.860086785Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_f6c2c6c4-927e-496d-873b-0e47ead3c40a","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.864854787Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_be258c33-8299-48b4-94e9-f6c15d8dd8a3","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.869396825Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_bb4b3cdb-0d2e-44b8-994b-0908726ddaeb","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:01.874002343Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:02.266519089Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"298.884705ms"} {"msg":"Profile for oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:05.081355518Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:05.081502892Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_f6c2c6c4-927e-496d-873b-0e47ead3c40a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:05.090789431Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:05.091144699Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_f6c2c6c4-927e-496d-873b-0e47ead3c40a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_be258c33-8299-48b4-94e9-f6c15d8dd8a3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:05.096378233Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:05.096724301Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:05.096736831Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_be258c33-8299-48b4-94e9-f6c15d8dd8a3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:05.101505154Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_bb4b3cdb-0d2e-44b8-994b-0908726ddaeb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:05.145706857Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:05.14581481Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_bb4b3cdb-0d2e-44b8-994b-0908726ddaeb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_f6c2c6c4-927e-496d-873b-0e47ead3c40a\") failed: Property not found. retry in 31.044842ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:05.230432187Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_f6c2c6c4-927e-496d-873b-0e47ead3c40a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e\") failed: Property not found. retry in 39.230567ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:05.238253002Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_be258c33-8299-48b4-94e9-f6c15d8dd8a3\") failed: Property not found. retry in 54.765739ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:05.244506989Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_be258c33-8299-48b4-94e9-f6c15d8dd8a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7\") failed: Property not found. retry in 59.066471ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:05.251338151Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f6c2c6c4-927e-496d-873b-0e47ead3c40a\") failed: Property not found. retry in 51.53516ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:05.268814843Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_f6c2c6c4-927e-496d-873b-0e47ead3c40a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e\") failed: Property not found. retry in 61.897202ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:05.285847065Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bb4b3cdb-0d2e-44b8-994b-0908726ddaeb\") failed: Property not found. retry in 63.156367ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:05.305017588Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_bb4b3cdb-0d2e-44b8-994b-0908726ddaeb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_be258c33-8299-48b4-94e9-f6c15d8dd8a3\") failed: Property not found. retry in 123.359354ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:05.634301002Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_be258c33-8299-48b4-94e9-f6c15d8dd8a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bb4b3cdb-0d2e-44b8-994b-0908726ddaeb\") failed: Property not found. retry in 103.626219ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:06.22514722Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_bb4b3cdb-0d2e-44b8-994b-0908726ddaeb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7\") failed: Property not found. retry in 123.177716ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:06.26033771Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e\") failed: Property not found. retry in 159.151099ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:06.647085651Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f6c2c6c4-927e-496d-873b-0e47ead3c40a\") failed: Property not found. retry in 140.386477ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:07.41332446Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_f6c2c6c4-927e-496d-873b-0e47ead3c40a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_be258c33-8299-48b4-94e9-f6c15d8dd8a3\") failed: Property not found. retry in 279.512364ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:08.005211883Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_be258c33-8299-48b4-94e9-f6c15d8dd8a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bb4b3cdb-0d2e-44b8-994b-0908726ddaeb\") failed: Property not found. retry in 127.0266ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:08.032022895Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_bb4b3cdb-0d2e-44b8-994b-0908726ddaeb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7\") failed: Property not found. retry in 217.44493ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:08.047408669Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e\") failed: Property not found. retry in 534.132131ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:08.088406307Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f6c2c6c4-927e-496d-873b-0e47ead3c40a\") failed: Property not found. retry in 586.872905ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:08.100563344Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_f6c2c6c4-927e-496d-873b-0e47ead3c40a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bb4b3cdb-0d2e-44b8-994b-0908726ddaeb\") failed: Property not found. retry in 478.794682ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:08.169428499Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_bb4b3cdb-0d2e-44b8-994b-0908726ddaeb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7\") failed: Property not found. retry in 435.275302ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:08.2749622Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_be258c33-8299-48b4-94e9-f6c15d8dd8a3\") failed: Property not found. retry in 498.401952ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:08.291296216Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_be258c33-8299-48b4-94e9-f6c15d8dd8a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e\") failed: Property not found. retry in 834.848307ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:08.633322581Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bb4b3cdb-0d2e-44b8-994b-0908726ddaeb\") failed: Property not found. retry in 949.252099ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:08.659494868Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_bb4b3cdb-0d2e-44b8-994b-0908726ddaeb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f6c2c6c4-927e-496d-873b-0e47ead3c40a\") failed: Property not found. retry in 1.170408452s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:08.698757375Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_f6c2c6c4-927e-496d-873b-0e47ead3c40a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7\") failed: Property not found. retry in 425.790935ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:08.722139697Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_be258c33-8299-48b4-94e9-f6c15d8dd8a3\") failed: Property not found. retry in 992.098425ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:08.801847109Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_be258c33-8299-48b4-94e9-f6c15d8dd8a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7\") failed: Property not found. retry in 637.564093ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:09.15918799Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e\") failed: Property not found. retry in 674.514295ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:09.479366163Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bb4b3cdb-0d2e-44b8-994b-0908726ddaeb\") failed: Property not found. retry in 973.983182ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:09.621606811Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_bb4b3cdb-0d2e-44b8-994b-0908726ddaeb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_be258c33-8299-48b4-94e9-f6c15d8dd8a3\") failed: Property not found. retry in 617.401849ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:09.802908741Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_be258c33-8299-48b4-94e9-f6c15d8dd8a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7\") failed: Property not found. retry in 1.008742201s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:09.809919997Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f6c2c6c4-927e-496d-873b-0e47ead3c40a\") failed: Property not found. retry in 727.6782ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:09.876515459Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_f6c2c6c4-927e-496d-873b-0e47ead3c40a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e\") failed: Property not found. retry in 966.493554ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:10.16080717Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_be258c33-8299-48b4-94e9-f6c15d8dd8a3\") failed: Property not found. retry in 1.16171715s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:10.428321945Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_be258c33-8299-48b4-94e9-f6c15d8dd8a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bb4b3cdb-0d2e-44b8-994b-0908726ddaeb\") failed: Property not found. retry in 826.588621ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:10.603292586Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_bb4b3cdb-0d2e-44b8-994b-0908726ddaeb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f6c2c6c4-927e-496d-873b-0e47ead3c40a\") failed: Property not found. retry in 719.070271ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:10.612648387Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_f6c2c6c4-927e-496d-873b-0e47ead3c40a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7\") failed: Property not found. retry in 1.002395149s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:10.826395553Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e\") failed: Property not found. retry in 991.025614ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:11.134423125Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f6c2c6c4-927e-496d-873b-0e47ead3c40a\") failed: Property not found. retry in 1.445613324s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:11.342542407Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_f6c2c6c4-927e-496d-873b-0e47ead3c40a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bb4b3cdb-0d2e-44b8-994b-0908726ddaeb\") failed: Property not found. retry in 763.300911ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:11.439557658Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_bb4b3cdb-0d2e-44b8-994b-0908726ddaeb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_be258c33-8299-48b4-94e9-f6c15d8dd8a3\") failed: Property not found. retry in 928.808749ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:11.599604666Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_be258c33-8299-48b4-94e9-f6c15d8dd8a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7\") failed: Property not found. retry in 1.028792046s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:11.838605598Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e\") failed: Property not found. retry in 1.136507653s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:12.135637531Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bb4b3cdb-0d2e-44b8-994b-0908726ddaeb\") failed: Property not found. retry in 1.069792897s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:12.212125287Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_bb4b3cdb-0d2e-44b8-994b-0908726ddaeb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_be258c33-8299-48b4-94e9-f6c15d8dd8a3\") failed: Property not found. retry in 744.952761ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:12.538723357Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_be258c33-8299-48b4-94e9-f6c15d8dd8a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7\") failed: Property not found. retry in 923.597715ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:12.878243782Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e\") failed: Property not found. retry in 1.1622914s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:13.281603874Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7\") failed: Property not found. retry in 816.68717ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:13.812957648Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e\") failed: Property not found. retry in 1.115247491s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:14.454187395Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7\") failed: Property not found. retry in 1.268619324s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:14.639316646Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e\") failed: Property not found. retry in 1.050094078s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:15.898758858Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7\") failed: Property not found. retry in 789.584821ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:15.927379743Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7\") failed: Property not found. retry in 774.074393ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:16.72951804Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e\") failed: Property not found. retry in 671.397386ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:16.962037339Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7\") failed: Property not found. retry in 847.532214ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:17.514215474Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e\") failed: Property not found. retry in 1.055510631s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:17.645087364Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:17.835032448Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"429.884712ms"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:17.886225806Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52162"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:17.886314009Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"0c61f052-e0b2-45a0-91bd-b17470a313b0","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52162","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":42,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7\") failed: Property not found. retry in 671.221195ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:18.681230325Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e\") failed: Property not found. retry in 730.724448ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:18.712202836Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7\") failed: Property not found. retry in 994.581184ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:19.364253778Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e\") failed: Property not found. retry in 1.167667479s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:19.454592641Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7\") failed: Property not found. retry in 1.048270232s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:20.368697811Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e\") failed: Property not found. retry in 741.981409ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:20.632189222Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e\") failed: Property not found. retry in 1.377465439s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:21.384326587Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7\") failed: Property not found. retry in 827.421778ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:21.426588475Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7\") failed: Property not found. retry in 773.087629ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:22.264192489Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e\") failed: Property not found. retry in 953.848397ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:22.771178897Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:23.747513266Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:23.747566727Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_cockroachdb_d6f58b5e-cabb-496a-9b02-c2e904c1eb7e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:23.747574347Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_cockroachdb_f6c2c6c4-927e-496d-873b-0e47ead3c40a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:23.747583257Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_cockroachdb_be258c33-8299-48b4-94e9-f6c15d8dd8a3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:23.747597158Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_cockroachdb_bb4b3cdb-0d2e-44b8-994b-0908726ddaeb"} {"msg":"Writing ledger to /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:23.747644839Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:23.748038128Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:23.748347806Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"adee70ef-40f7-4575-874e-e905cdf6cc0b","remote_addr":"[fd00:1122:3344:101::1]:57929","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":22168112,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:23.779959122Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:34351"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:23.807676136Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"d50524ff-f0c8-4ed3-b6a3-f44508ec666f","remote_addr":"[fd00:1122:3344:101::1]:34351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":27549,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:23.807919002Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959","retry_after":"69.307067ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d50524ff-f0c8-4ed3-b6a3-f44508ec666f\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:49:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d50524ff-f0c8-4ed3-b6a3-f44508ec666f\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:23.904542953Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"1fa40766-c0c1-4271-b3b4-82eadd2181c6","remote_addr":"[fd00:1122:3344:101::1]:34351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":25397,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:23.904735767Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959","retry_after":"100.523735ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1fa40766-c0c1-4271-b3b4-82eadd2181c6\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:49:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1fa40766-c0c1-4271-b3b4-82eadd2181c6\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:24.032518384Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"f7348f03-0fd7-4d55-8f05-f49fda4c8268","remote_addr":"[fd00:1122:3344:101::1]:34351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":25507,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:24.032706328Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959","retry_after":"186.149034ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f7348f03-0fd7-4d55-8f05-f49fda4c8268\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:49:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f7348f03-0fd7-4d55-8f05-f49fda4c8268\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:24.243630533Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"75dd6025-ed19-4726-82d6-9d7de0ba8788","remote_addr":"[fd00:1122:3344:101::1]:34351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":23688,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:24.243771874Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959","retry_after":"90.155768ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"75dd6025-ed19-4726-82d6-9d7de0ba8788\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:49:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"75dd6025-ed19-4726-82d6-9d7de0ba8788\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:24.358896479Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"bdbe6eb4-56ac-4241-9b32-d826106453c1","remote_addr":"[fd00:1122:3344:101::1]:34351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":23969,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:24.359051112Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959","retry_after":"103.820711ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bdbe6eb4-56ac-4241-9b32-d826106453c1\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:49:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bdbe6eb4-56ac-4241-9b32-d826106453c1\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:24.490507316Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"f025b047-56d9-4369-88ed-dc28128e601a","remote_addr":"[fd00:1122:3344:101::1]:34351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":27269,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:24.49068225Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959","retry_after":"189.688744ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f025b047-56d9-4369-88ed-dc28128e601a\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:49:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f025b047-56d9-4369-88ed-dc28128e601a\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:24.705680806Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"3918fa11-cb36-49b5-88ae-319a323ae4f5","remote_addr":"[fd00:1122:3344:101::1]:34351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":23783,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:24.7058677Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959","retry_after":"183.556163ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3918fa11-cb36-49b5-88ae-319a323ae4f5\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:49:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3918fa11-cb36-49b5-88ae-319a323ae4f5\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:24.916478112Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"4c213488-a44b-41c8-85bb-d0279459fd95","remote_addr":"[fd00:1122:3344:101::1]:34351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_b46b9748-75d3-4248-95bb-b19942b724b7 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":25447,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:49:24.916668456Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959","retry_after":"302.43246ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4c213488-a44b-41c8-85bb-d0279459fd95\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:49:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4c213488-a44b-41c8-85bb-d0279459fd95\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::6]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:25.248965231Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3465"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:49:36.753594885Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3483"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:10.977987319Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3506"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:10.978713946Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"eb84fbea-90ca-4879-897e-c6b537101d89","remote_addr":"[fd00:1122:3344:101::1]:34351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45751170,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:10.97961385Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55712"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:10.986487626Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"896.644525ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6fc5bc0c-95b6-40a3-a5a9-955d74c48961, kind: Service { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee }, name: Name(\"ntp-f73788bd-d6b7-4d17-b028-bd04b6dd3aee\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: b46b9748-75d3-4248-95bb-b19942b724b7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: b605874a-c92a-4924-97a2-40d87990c249, kind: Service { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175 }, name: Name(\"external-dns-4ab1258a-9fc9-4e50-a387-e0cecec3f175\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 168])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 55f61404-8e20-4047-a779-ffe87d352f7e, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 68e120cd-095b-4827-9d1b-7b977952fe73, kind: Service { id: 55f61404-8e20-4047-a779-ffe87d352f7e }, name: Name(\"external-dns-55f61404-8e20-4047-a779-ffe87d352f7e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 7])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ec0c20b-4307-47e1-97b9-77d1ea3d29b3, kind: Service { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc }, name: Name(\"nexus-35af6b86-fd74-4104-b900-7a3edf8d03dc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 165, 129])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: d6d27046-4d61-4670-9a2f-1ffbb00db647, kind: Service { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599 }, name: Name(\"nexus-73e067cf-cfe8-480d-9d5d-fff7aeec0599\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 89])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 50145656-8d32-471a-8a33-3c47dc24d267, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8e49e841-d58f-44ba-beb8-68b0b0f8fc4c, kind: Service { id: 50145656-8d32-471a-8a33-3c47dc24d267 }, name: Name(\"nexus-50145656-8d32-471a-8a33-3c47dc24d267\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 470307fd-e881-437d-ad9f-f54f87acee44, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e05c51a-6602-4ef4-bc51-8d51552f353c, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 5f758d69-a06e-4648-b5fe-686e96c65857, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e996ca7-157f-4df1-ab30-04eac3863eb1, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: 524c932e-94cf-424b-b974-cd7c6d21eda9, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a330eff-2e60-4b6c-95a2-192e89df3181, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External } } } }, OmicronZoneConfig { id: bc50e194-c3e3-4fbb-a14d-26741c36b466, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.01630472Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.016804374Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:50078"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.01806845Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.018074487Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.043646806Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.043702185Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.063516872Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.063561339Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.081968512Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.082007663Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.100290413Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.10033428Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.117726984Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.117766548Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.135561339Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.135607184Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.153337124Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.153379341Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.171063256Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.171104179Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55f61404-8e20-4047-a779-ffe87d352f7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.233044595Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.233087965Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55f61404-8e20-4047-a779-ffe87d352f7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.292442103Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.292497895Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.347307743Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.347352734Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.398288338Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.398334229Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.450778056Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.450820273Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 524c932e-94cf-424b-b974-cd7c6d21eda9, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.511881359Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.511928039Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 524c932e-94cf-424b-b974-cd7c6d21eda9, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.570246439Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.570296305Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a330eff-2e60-4b6c-95a2-192e89df3181, dataset_name: DatasetName { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.627233126Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.627281549Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a330eff-2e60-4b6c-95a2-192e89df3181, dataset_name: DatasetName { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc50e194-c3e3-4fbb-a14d-26741c36b466, dataset_name: DatasetName { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.682712984Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.682753719Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bc50e194-c3e3-4fbb-a14d-26741c36b466, dataset_name: DatasetName { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, dataset_name: DatasetName { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.739734068Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.739786195Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, dataset_name: DatasetName { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, dataset_name: DatasetName { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.799836348Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.799882981Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, dataset_name: DatasetName { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.85815379Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.858202578Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.85848601Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.858751741Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:11.965429816Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:11.965474948Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:11.965479514Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:11.965483668Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:11.965487643Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:11.96549083Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:11.96549488Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:11.965499005Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:11.965502193Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:11.965506233Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:11.965510152Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:11.965521656Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:11.965526025Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:11.965529025Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:11.965532981Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:11.965536909Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:11.965540837Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:11.965544719Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:11.965548506Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:11.966254351Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"f496e683-8931-4b6c-8b63-40500c499a05","remote_addr":"[fd00:1122:3344:101::1]:50078","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\", TimeNotSynchronized), (\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\", TimeNotSynchronized), (\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\", TimeNotSynchronized), (\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\", TimeNotSynchronized), (\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\", TimeNotSynchronized), (\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\", TimeNotSynchronized), (\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\", TimeNotSynchronized), (\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\", TimeNotSynchronized), (\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\", TimeNotSynchronized), (\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\", TimeNotSynchronized), (\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\", TimeNotSynchronized), (\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\", TimeNotSynchronized), (\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\", TimeNotSynchronized), (\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\", TimeNotSynchronized), (\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\", TimeNotSynchronized), (\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\", TimeNotSynchronized), (\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\", TimeNotSynchronized), (\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\", TimeNotSynchronized)]","latency_us":946428,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:11.966611912Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"104.585811ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f496e683-8931-4b6c-8b63-40500c499a05\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:50:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f496e683-8931-4b6c-8b63-40500c499a05\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6fc5bc0c-95b6-40a3-a5a9-955d74c48961, kind: Service { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee }, name: Name(\"ntp-f73788bd-d6b7-4d17-b028-bd04b6dd3aee\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: b46b9748-75d3-4248-95bb-b19942b724b7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: b605874a-c92a-4924-97a2-40d87990c249, kind: Service { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175 }, name: Name(\"external-dns-4ab1258a-9fc9-4e50-a387-e0cecec3f175\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 168])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 55f61404-8e20-4047-a779-ffe87d352f7e, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 68e120cd-095b-4827-9d1b-7b977952fe73, kind: Service { id: 55f61404-8e20-4047-a779-ffe87d352f7e }, name: Name(\"external-dns-55f61404-8e20-4047-a779-ffe87d352f7e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 7])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ec0c20b-4307-47e1-97b9-77d1ea3d29b3, kind: Service { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc }, name: Name(\"nexus-35af6b86-fd74-4104-b900-7a3edf8d03dc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 165, 129])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: d6d27046-4d61-4670-9a2f-1ffbb00db647, kind: Service { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599 }, name: Name(\"nexus-73e067cf-cfe8-480d-9d5d-fff7aeec0599\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 89])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 50145656-8d32-471a-8a33-3c47dc24d267, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8e49e841-d58f-44ba-beb8-68b0b0f8fc4c, kind: Service { id: 50145656-8d32-471a-8a33-3c47dc24d267 }, name: Name(\"nexus-50145656-8d32-471a-8a33-3c47dc24d267\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 470307fd-e881-437d-ad9f-f54f87acee44, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e05c51a-6602-4ef4-bc51-8d51552f353c, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 5f758d69-a06e-4648-b5fe-686e96c65857, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e996ca7-157f-4df1-ab30-04eac3863eb1, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: 524c932e-94cf-424b-b974-cd7c6d21eda9, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a330eff-2e60-4b6c-95a2-192e89df3181, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External } } } }, OmicronZoneConfig { id: bc50e194-c3e3-4fbb-a14d-26741c36b466, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.072027639Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.07249547Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.072501527Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.091574414Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.091619378Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.110590283Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.110627213Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.130293885Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.130332409Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.148518304Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.148555036Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.166873242Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.166911915Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.184713616Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.184751379Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.20276409Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.202812335Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.220765717Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.220808327Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55f61404-8e20-4047-a779-ffe87d352f7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.238818298Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.238859099Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55f61404-8e20-4047-a779-ffe87d352f7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.257080205Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.257121474Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.274949038Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.274988283Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.292915492Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.292950546Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.310708192Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.310742945Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 524c932e-94cf-424b-b974-cd7c6d21eda9, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.328497864Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.32853423Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 524c932e-94cf-424b-b974-cd7c6d21eda9, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.346603154Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.346644143Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a330eff-2e60-4b6c-95a2-192e89df3181, dataset_name: DatasetName { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.364663077Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.364698807Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a330eff-2e60-4b6c-95a2-192e89df3181, dataset_name: DatasetName { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc50e194-c3e3-4fbb-a14d-26741c36b466, dataset_name: DatasetName { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.382422979Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.382471712Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bc50e194-c3e3-4fbb-a14d-26741c36b466, dataset_name: DatasetName { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, dataset_name: DatasetName { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.400191508Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.400229609Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, dataset_name: DatasetName { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, dataset_name: DatasetName { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.417878674Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.417917479Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, dataset_name: DatasetName { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.436186965Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.436305153Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.436541512Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.436695359Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:12.527715259Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:12.527756266Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:12.527763447Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:12.527769813Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:12.527774744Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:12.52778051Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:12.527785592Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:12.527791592Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:12.527796411Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:12.527801379Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:12.527806301Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:12.527826552Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:12.527832093Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:12.52783738Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:12.527842265Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:12.52784804Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:12.527853852Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:12.52785868Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:12.527864222Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.527983643Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"be72478d-038e-4497-857e-f6e57660b3f3","remote_addr":"[fd00:1122:3344:101::1]:50078","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\", TimeNotSynchronized), (\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\", TimeNotSynchronized), (\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\", TimeNotSynchronized), (\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\", TimeNotSynchronized), (\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\", TimeNotSynchronized), (\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\", TimeNotSynchronized), (\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\", TimeNotSynchronized), (\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\", TimeNotSynchronized), (\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\", TimeNotSynchronized), (\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\", TimeNotSynchronized), (\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\", TimeNotSynchronized), (\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\", TimeNotSynchronized), (\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\", TimeNotSynchronized), (\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\", TimeNotSynchronized), (\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\", TimeNotSynchronized), (\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\", TimeNotSynchronized), (\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\", TimeNotSynchronized), (\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\", TimeNotSynchronized)]","latency_us":454504,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:12.528159963Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"154.92459ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"be72478d-038e-4497-857e-f6e57660b3f3\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:50:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"be72478d-038e-4497-857e-f6e57660b3f3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6fc5bc0c-95b6-40a3-a5a9-955d74c48961, kind: Service { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee }, name: Name(\"ntp-f73788bd-d6b7-4d17-b028-bd04b6dd3aee\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: b46b9748-75d3-4248-95bb-b19942b724b7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: b605874a-c92a-4924-97a2-40d87990c249, kind: Service { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175 }, name: Name(\"external-dns-4ab1258a-9fc9-4e50-a387-e0cecec3f175\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 168])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 55f61404-8e20-4047-a779-ffe87d352f7e, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 68e120cd-095b-4827-9d1b-7b977952fe73, kind: Service { id: 55f61404-8e20-4047-a779-ffe87d352f7e }, name: Name(\"external-dns-55f61404-8e20-4047-a779-ffe87d352f7e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 7])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ec0c20b-4307-47e1-97b9-77d1ea3d29b3, kind: Service { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc }, name: Name(\"nexus-35af6b86-fd74-4104-b900-7a3edf8d03dc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 165, 129])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: d6d27046-4d61-4670-9a2f-1ffbb00db647, kind: Service { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599 }, name: Name(\"nexus-73e067cf-cfe8-480d-9d5d-fff7aeec0599\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 89])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 50145656-8d32-471a-8a33-3c47dc24d267, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8e49e841-d58f-44ba-beb8-68b0b0f8fc4c, kind: Service { id: 50145656-8d32-471a-8a33-3c47dc24d267 }, name: Name(\"nexus-50145656-8d32-471a-8a33-3c47dc24d267\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 470307fd-e881-437d-ad9f-f54f87acee44, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e05c51a-6602-4ef4-bc51-8d51552f353c, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 5f758d69-a06e-4648-b5fe-686e96c65857, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e996ca7-157f-4df1-ab30-04eac3863eb1, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: 524c932e-94cf-424b-b974-cd7c6d21eda9, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a330eff-2e60-4b6c-95a2-192e89df3181, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External } } } }, OmicronZoneConfig { id: bc50e194-c3e3-4fbb-a14d-26741c36b466, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.684721693Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.685079662Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.685086956Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.705338146Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.705374222Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.723777547Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.723802222Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.743262586Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.743297199Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.761496987Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.761522179Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.779941131Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.779974207Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.797705283Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.797732687Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.815034435Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.815065289Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.831994864Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.83202254Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55f61404-8e20-4047-a779-ffe87d352f7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.850919009Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.850945137Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55f61404-8e20-4047-a779-ffe87d352f7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.869097674Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.8691393Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.888277312Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.888308766Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.907175705Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.907203446Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.925653323Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.925679048Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 524c932e-94cf-424b-b974-cd7c6d21eda9, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.943450094Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.943478294Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 524c932e-94cf-424b-b974-cd7c6d21eda9, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.961280484Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.961309369Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a330eff-2e60-4b6c-95a2-192e89df3181, dataset_name: DatasetName { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.980001538Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.980028613Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a330eff-2e60-4b6c-95a2-192e89df3181, dataset_name: DatasetName { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc50e194-c3e3-4fbb-a14d-26741c36b466, dataset_name: DatasetName { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.998124407Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:12.998152121Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bc50e194-c3e3-4fbb-a14d-26741c36b466, dataset_name: DatasetName { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, dataset_name: DatasetName { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.016007763Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.016040529Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, dataset_name: DatasetName { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, dataset_name: DatasetName { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.033899699Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.033924347Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, dataset_name: DatasetName { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.051648739Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.051685218Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.052007131Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.052157866Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.142803539Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.142852524Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.142859847Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.142866288Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.142872738Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.142879038Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.142885394Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.142891122Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.142895979Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.142901604Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.14290886Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.142925257Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.142930929Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.142937924Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.142943464Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.142949014Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.142956102Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.142962064Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.142967624Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.143089305Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"61d0b58e-ecb7-431c-bab5-17e77b5076b7","remote_addr":"[fd00:1122:3344:101::1]:50078","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\", TimeNotSynchronized), (\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\", TimeNotSynchronized), (\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\", TimeNotSynchronized), (\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\", TimeNotSynchronized), (\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\", TimeNotSynchronized), (\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\", TimeNotSynchronized), (\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\", TimeNotSynchronized), (\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\", TimeNotSynchronized), (\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\", TimeNotSynchronized), (\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\", TimeNotSynchronized), (\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\", TimeNotSynchronized), (\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\", TimeNotSynchronized), (\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\", TimeNotSynchronized), (\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\", TimeNotSynchronized), (\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\", TimeNotSynchronized), (\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\", TimeNotSynchronized), (\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\", TimeNotSynchronized), (\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\", TimeNotSynchronized)]","latency_us":456991,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.143367896Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"86.396805ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"61d0b58e-ecb7-431c-bab5-17e77b5076b7\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:50:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"61d0b58e-ecb7-431c-bab5-17e77b5076b7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6fc5bc0c-95b6-40a3-a5a9-955d74c48961, kind: Service { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee }, name: Name(\"ntp-f73788bd-d6b7-4d17-b028-bd04b6dd3aee\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: b46b9748-75d3-4248-95bb-b19942b724b7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: b605874a-c92a-4924-97a2-40d87990c249, kind: Service { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175 }, name: Name(\"external-dns-4ab1258a-9fc9-4e50-a387-e0cecec3f175\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 168])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 55f61404-8e20-4047-a779-ffe87d352f7e, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 68e120cd-095b-4827-9d1b-7b977952fe73, kind: Service { id: 55f61404-8e20-4047-a779-ffe87d352f7e }, name: Name(\"external-dns-55f61404-8e20-4047-a779-ffe87d352f7e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 7])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ec0c20b-4307-47e1-97b9-77d1ea3d29b3, kind: Service { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc }, name: Name(\"nexus-35af6b86-fd74-4104-b900-7a3edf8d03dc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 165, 129])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: d6d27046-4d61-4670-9a2f-1ffbb00db647, kind: Service { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599 }, name: Name(\"nexus-73e067cf-cfe8-480d-9d5d-fff7aeec0599\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 89])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 50145656-8d32-471a-8a33-3c47dc24d267, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8e49e841-d58f-44ba-beb8-68b0b0f8fc4c, kind: Service { id: 50145656-8d32-471a-8a33-3c47dc24d267 }, name: Name(\"nexus-50145656-8d32-471a-8a33-3c47dc24d267\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 470307fd-e881-437d-ad9f-f54f87acee44, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e05c51a-6602-4ef4-bc51-8d51552f353c, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 5f758d69-a06e-4648-b5fe-686e96c65857, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e996ca7-157f-4df1-ab30-04eac3863eb1, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: 524c932e-94cf-424b-b974-cd7c6d21eda9, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a330eff-2e60-4b6c-95a2-192e89df3181, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External } } } }, OmicronZoneConfig { id: bc50e194-c3e3-4fbb-a14d-26741c36b466, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.231737909Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.232024538Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.232032441Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.250356368Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.250398227Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.26876539Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.26880276Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.287936261Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.287974898Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.306854281Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.306888453Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.322397389Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.322443196Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.339467939Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.339507933Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.358296375Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.358328017Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.375724148Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.375759746Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55f61404-8e20-4047-a779-ffe87d352f7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.393993513Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.394028042Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55f61404-8e20-4047-a779-ffe87d352f7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.412064366Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.412105223Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.430538897Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.43057846Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.448118912Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.448162656Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.466486806Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.46652171Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 524c932e-94cf-424b-b974-cd7c6d21eda9, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.484163347Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.484201579Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 524c932e-94cf-424b-b974-cd7c6d21eda9, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.501277091Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.501321239Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a330eff-2e60-4b6c-95a2-192e89df3181, dataset_name: DatasetName { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.51934005Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.519377157Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a330eff-2e60-4b6c-95a2-192e89df3181, dataset_name: DatasetName { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc50e194-c3e3-4fbb-a14d-26741c36b466, dataset_name: DatasetName { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.536633838Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.536674789Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bc50e194-c3e3-4fbb-a14d-26741c36b466, dataset_name: DatasetName { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, dataset_name: DatasetName { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.555684542Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.555717439Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, dataset_name: DatasetName { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, dataset_name: DatasetName { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.573283486Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.573389584Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, dataset_name: DatasetName { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.590501326Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.590544424Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.59091272Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.591061814Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.681683053Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.681733914Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.681742333Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.681748643Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.681754718Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.68176158Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.681768986Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.681776253Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.681782253Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.681789575Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.681814456Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.681822257Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.681829476Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.681836807Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.681842751Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.681849942Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.681858464Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.681874158Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.681889627Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.682078903Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"76056a9c-7f4b-41f1-bbac-cc0430302b54","remote_addr":"[fd00:1122:3344:101::1]:50078","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\", TimeNotSynchronized), (\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\", TimeNotSynchronized), (\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\", TimeNotSynchronized), (\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\", TimeNotSynchronized), (\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\", TimeNotSynchronized), (\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\", TimeNotSynchronized), (\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\", TimeNotSynchronized), (\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\", TimeNotSynchronized), (\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\", TimeNotSynchronized), (\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\", TimeNotSynchronized), (\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\", TimeNotSynchronized), (\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\", TimeNotSynchronized), (\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\", TimeNotSynchronized), (\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\", TimeNotSynchronized), (\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\", TimeNotSynchronized), (\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\", TimeNotSynchronized), (\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\", TimeNotSynchronized), (\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\", TimeNotSynchronized)]","latency_us":448977,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:13.68229249Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"208.280489ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"76056a9c-7f4b-41f1-bbac-cc0430302b54\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:50:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"76056a9c-7f4b-41f1-bbac-cc0430302b54\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6fc5bc0c-95b6-40a3-a5a9-955d74c48961, kind: Service { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee }, name: Name(\"ntp-f73788bd-d6b7-4d17-b028-bd04b6dd3aee\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: b46b9748-75d3-4248-95bb-b19942b724b7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: b605874a-c92a-4924-97a2-40d87990c249, kind: Service { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175 }, name: Name(\"external-dns-4ab1258a-9fc9-4e50-a387-e0cecec3f175\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 168])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 55f61404-8e20-4047-a779-ffe87d352f7e, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 68e120cd-095b-4827-9d1b-7b977952fe73, kind: Service { id: 55f61404-8e20-4047-a779-ffe87d352f7e }, name: Name(\"external-dns-55f61404-8e20-4047-a779-ffe87d352f7e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 7])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ec0c20b-4307-47e1-97b9-77d1ea3d29b3, kind: Service { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc }, name: Name(\"nexus-35af6b86-fd74-4104-b900-7a3edf8d03dc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 165, 129])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: d6d27046-4d61-4670-9a2f-1ffbb00db647, kind: Service { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599 }, name: Name(\"nexus-73e067cf-cfe8-480d-9d5d-fff7aeec0599\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 89])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 50145656-8d32-471a-8a33-3c47dc24d267, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8e49e841-d58f-44ba-beb8-68b0b0f8fc4c, kind: Service { id: 50145656-8d32-471a-8a33-3c47dc24d267 }, name: Name(\"nexus-50145656-8d32-471a-8a33-3c47dc24d267\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 470307fd-e881-437d-ad9f-f54f87acee44, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e05c51a-6602-4ef4-bc51-8d51552f353c, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 5f758d69-a06e-4648-b5fe-686e96c65857, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e996ca7-157f-4df1-ab30-04eac3863eb1, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: 524c932e-94cf-424b-b974-cd7c6d21eda9, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a330eff-2e60-4b6c-95a2-192e89df3181, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External } } } }, OmicronZoneConfig { id: bc50e194-c3e3-4fbb-a14d-26741c36b466, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.892328227Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.893262469Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.893268544Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.916800366Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.916839095Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.93762965Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.937663194Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.956514939Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.956552103Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.974843789Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.974877353Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.996634672Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:13.996676992Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.016246313Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.016277327Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.036893644Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.036925155Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.056105843Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.0561395Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55f61404-8e20-4047-a779-ffe87d352f7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.07487391Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.074903478Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55f61404-8e20-4047-a779-ffe87d352f7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.094976958Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.095026075Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.114418559Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.114450641Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.132592566Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.132634951Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.151164461Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.15120739Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 524c932e-94cf-424b-b974-cd7c6d21eda9, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.169317081Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.169346115Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 524c932e-94cf-424b-b974-cd7c6d21eda9, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.187235997Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.187285432Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a330eff-2e60-4b6c-95a2-192e89df3181, dataset_name: DatasetName { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.204936534Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.204970144Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a330eff-2e60-4b6c-95a2-192e89df3181, dataset_name: DatasetName { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc50e194-c3e3-4fbb-a14d-26741c36b466, dataset_name: DatasetName { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.222410457Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.222447508Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bc50e194-c3e3-4fbb-a14d-26741c36b466, dataset_name: DatasetName { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, dataset_name: DatasetName { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.241308302Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.241346449Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, dataset_name: DatasetName { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, dataset_name: DatasetName { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.259991645Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.260033562Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, dataset_name: DatasetName { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.278656353Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.278692391Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.278925515Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.279068769Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:14.369931326Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:14.369992012Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:14.370003956Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:14.370013106Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:14.370021919Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:14.370030554Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:14.370037632Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:14.370044298Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:14.370052905Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:14.370061511Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:14.370081208Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:14.37008853Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:14.370094239Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:14.370103006Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:14.370108199Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:14.370115184Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:14.370123884Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:14.370128834Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:14.370133944Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.370188873Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"49a2dbf5-b782-4405-ac4b-604e2fb9ecd3","remote_addr":"[fd00:1122:3344:101::1]:50078","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\", TimeNotSynchronized), (\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\", TimeNotSynchronized), (\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\", TimeNotSynchronized), (\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\", TimeNotSynchronized), (\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\", TimeNotSynchronized), (\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\", TimeNotSynchronized), (\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\", TimeNotSynchronized), (\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\", TimeNotSynchronized), (\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\", TimeNotSynchronized), (\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\", TimeNotSynchronized), (\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\", TimeNotSynchronized), (\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\", TimeNotSynchronized), (\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\", TimeNotSynchronized), (\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\", TimeNotSynchronized), (\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\", TimeNotSynchronized), (\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\", TimeNotSynchronized), (\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\", TimeNotSynchronized), (\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\", TimeNotSynchronized)]","latency_us":476426,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:14.370397097Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"219.1691ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"49a2dbf5-b782-4405-ac4b-604e2fb9ecd3\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:50:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"49a2dbf5-b782-4405-ac4b-604e2fb9ecd3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6fc5bc0c-95b6-40a3-a5a9-955d74c48961, kind: Service { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee }, name: Name(\"ntp-f73788bd-d6b7-4d17-b028-bd04b6dd3aee\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: b46b9748-75d3-4248-95bb-b19942b724b7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: b605874a-c92a-4924-97a2-40d87990c249, kind: Service { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175 }, name: Name(\"external-dns-4ab1258a-9fc9-4e50-a387-e0cecec3f175\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 168])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 55f61404-8e20-4047-a779-ffe87d352f7e, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 68e120cd-095b-4827-9d1b-7b977952fe73, kind: Service { id: 55f61404-8e20-4047-a779-ffe87d352f7e }, name: Name(\"external-dns-55f61404-8e20-4047-a779-ffe87d352f7e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 7])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ec0c20b-4307-47e1-97b9-77d1ea3d29b3, kind: Service { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc }, name: Name(\"nexus-35af6b86-fd74-4104-b900-7a3edf8d03dc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 165, 129])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: d6d27046-4d61-4670-9a2f-1ffbb00db647, kind: Service { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599 }, name: Name(\"nexus-73e067cf-cfe8-480d-9d5d-fff7aeec0599\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 89])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 50145656-8d32-471a-8a33-3c47dc24d267, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8e49e841-d58f-44ba-beb8-68b0b0f8fc4c, kind: Service { id: 50145656-8d32-471a-8a33-3c47dc24d267 }, name: Name(\"nexus-50145656-8d32-471a-8a33-3c47dc24d267\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 470307fd-e881-437d-ad9f-f54f87acee44, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e05c51a-6602-4ef4-bc51-8d51552f353c, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 5f758d69-a06e-4648-b5fe-686e96c65857, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e996ca7-157f-4df1-ab30-04eac3863eb1, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: 524c932e-94cf-424b-b974-cd7c6d21eda9, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a330eff-2e60-4b6c-95a2-192e89df3181, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External } } } }, OmicronZoneConfig { id: bc50e194-c3e3-4fbb-a14d-26741c36b466, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.59154064Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.592077119Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.59214881Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.612493129Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.612540689Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.631559453Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.631597357Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.650445345Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.650483324Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.668354579Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.668397958Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.686642368Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.686701103Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.704322368Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.704364463Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.722734733Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.722769393Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.740093954Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.740133189Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55f61404-8e20-4047-a779-ffe87d352f7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.759773894Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.759821155Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55f61404-8e20-4047-a779-ffe87d352f7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.778149337Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.778191478Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.796238512Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.796277279Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.814242107Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.814283162Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.832929898Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.832977261Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 524c932e-94cf-424b-b974-cd7c6d21eda9, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.851232421Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.851280066Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 524c932e-94cf-424b-b974-cd7c6d21eda9, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.869508327Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.86956094Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a330eff-2e60-4b6c-95a2-192e89df3181, dataset_name: DatasetName { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.888465649Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.88850959Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a330eff-2e60-4b6c-95a2-192e89df3181, dataset_name: DatasetName { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc50e194-c3e3-4fbb-a14d-26741c36b466, dataset_name: DatasetName { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.907070851Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.907104368Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bc50e194-c3e3-4fbb-a14d-26741c36b466, dataset_name: DatasetName { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, dataset_name: DatasetName { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.925504783Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.925544178Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, dataset_name: DatasetName { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, dataset_name: DatasetName { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.943579312Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.94362198Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, dataset_name: DatasetName { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.96170772Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.961783678Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.96195697Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:14.962068966Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.050320571Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.050428481Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.050435869Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.050441101Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.050447026Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.050453157Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.050459382Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.050464323Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.050470118Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.050476155Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.050500868Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.050507233Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.050513383Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.050518531Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.050524512Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.050528384Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.050534375Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.050539119Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.050545109Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.050686141Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"eb56c561-ca09-4144-ac59-d3942e34e964","remote_addr":"[fd00:1122:3344:101::1]:50078","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\", TimeNotSynchronized), (\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\", TimeNotSynchronized), (\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\", TimeNotSynchronized), (\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\", TimeNotSynchronized), (\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\", TimeNotSynchronized), (\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\", TimeNotSynchronized), (\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\", TimeNotSynchronized), (\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\", TimeNotSynchronized), (\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\", TimeNotSynchronized), (\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\", TimeNotSynchronized), (\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\", TimeNotSynchronized), (\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\", TimeNotSynchronized), (\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\", TimeNotSynchronized), (\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\", TimeNotSynchronized), (\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\", TimeNotSynchronized), (\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\", TimeNotSynchronized), (\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\", TimeNotSynchronized), (\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\", TimeNotSynchronized)]","latency_us":457651,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.050890502Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"293.004743ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"eb56c561-ca09-4144-ac59-d3942e34e964\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:50:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"eb56c561-ca09-4144-ac59-d3942e34e964\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6fc5bc0c-95b6-40a3-a5a9-955d74c48961, kind: Service { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee }, name: Name(\"ntp-f73788bd-d6b7-4d17-b028-bd04b6dd3aee\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: b46b9748-75d3-4248-95bb-b19942b724b7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: b605874a-c92a-4924-97a2-40d87990c249, kind: Service { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175 }, name: Name(\"external-dns-4ab1258a-9fc9-4e50-a387-e0cecec3f175\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 168])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 55f61404-8e20-4047-a779-ffe87d352f7e, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 68e120cd-095b-4827-9d1b-7b977952fe73, kind: Service { id: 55f61404-8e20-4047-a779-ffe87d352f7e }, name: Name(\"external-dns-55f61404-8e20-4047-a779-ffe87d352f7e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 7])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ec0c20b-4307-47e1-97b9-77d1ea3d29b3, kind: Service { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc }, name: Name(\"nexus-35af6b86-fd74-4104-b900-7a3edf8d03dc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 165, 129])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: d6d27046-4d61-4670-9a2f-1ffbb00db647, kind: Service { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599 }, name: Name(\"nexus-73e067cf-cfe8-480d-9d5d-fff7aeec0599\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 89])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 50145656-8d32-471a-8a33-3c47dc24d267, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8e49e841-d58f-44ba-beb8-68b0b0f8fc4c, kind: Service { id: 50145656-8d32-471a-8a33-3c47dc24d267 }, name: Name(\"nexus-50145656-8d32-471a-8a33-3c47dc24d267\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 470307fd-e881-437d-ad9f-f54f87acee44, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e05c51a-6602-4ef4-bc51-8d51552f353c, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 5f758d69-a06e-4648-b5fe-686e96c65857, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e996ca7-157f-4df1-ab30-04eac3863eb1, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: 524c932e-94cf-424b-b974-cd7c6d21eda9, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a330eff-2e60-4b6c-95a2-192e89df3181, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External } } } }, OmicronZoneConfig { id: bc50e194-c3e3-4fbb-a14d-26741c36b466, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.346079814Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.346432277Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.34643988Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.367565088Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.367606367Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.385787459Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.385815912Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.404405792Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.404444052Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.422636877Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.422666241Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.439741399Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.439793891Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.457875044Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.457921658Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.47625244Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.47627855Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.495211681Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.495249276Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55f61404-8e20-4047-a779-ffe87d352f7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.514141135Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.514182113Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55f61404-8e20-4047-a779-ffe87d352f7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.533554665Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.53358085Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.552012985Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.552042686Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.57040701Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.570441876Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.58903409Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.58906396Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 524c932e-94cf-424b-b974-cd7c6d21eda9, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.607360762Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.607389038Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 524c932e-94cf-424b-b974-cd7c6d21eda9, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.625769753Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.625805238Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a330eff-2e60-4b6c-95a2-192e89df3181, dataset_name: DatasetName { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.644346892Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.644378524Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a330eff-2e60-4b6c-95a2-192e89df3181, dataset_name: DatasetName { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc50e194-c3e3-4fbb-a14d-26741c36b466, dataset_name: DatasetName { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.662179319Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.662220195Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bc50e194-c3e3-4fbb-a14d-26741c36b466, dataset_name: DatasetName { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, dataset_name: DatasetName { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.680355329Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.680387494Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, dataset_name: DatasetName { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, dataset_name: DatasetName { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.698490737Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.698539572Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, dataset_name: DatasetName { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.716099541Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.716142517Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.716491304Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.716654752Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.812297062Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.812335003Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.812341809Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.812345991Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.81235005Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.812354831Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.812358759Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.812363438Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.81236761Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.81237886Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.812382995Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.812388207Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.81239311Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.812396176Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.812400114Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.812405383Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.812410314Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.812481151Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.812485529Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:15.81258199Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"470a56c8-710a-4dcc-b92f-395ef0c08977","remote_addr":"[fd00:1122:3344:101::1]:50078","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\", TimeNotSynchronized), (\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\", TimeNotSynchronized), (\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\", TimeNotSynchronized), (\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\", TimeNotSynchronized), (\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\", TimeNotSynchronized), (\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\", TimeNotSynchronized), (\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\", TimeNotSynchronized), (\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\", TimeNotSynchronized), (\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\", TimeNotSynchronized), (\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\", TimeNotSynchronized), (\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\", TimeNotSynchronized), (\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\", TimeNotSynchronized), (\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\", TimeNotSynchronized), (\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\", TimeNotSynchronized), (\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\", TimeNotSynchronized), (\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\", TimeNotSynchronized), (\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\", TimeNotSynchronized), (\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\", TimeNotSynchronized)]","latency_us":465046,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:15.812725872Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"350.476422ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"470a56c8-710a-4dcc-b92f-395ef0c08977\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:50:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"470a56c8-710a-4dcc-b92f-395ef0c08977\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6fc5bc0c-95b6-40a3-a5a9-955d74c48961, kind: Service { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee }, name: Name(\"ntp-f73788bd-d6b7-4d17-b028-bd04b6dd3aee\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: b46b9748-75d3-4248-95bb-b19942b724b7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: b605874a-c92a-4924-97a2-40d87990c249, kind: Service { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175 }, name: Name(\"external-dns-4ab1258a-9fc9-4e50-a387-e0cecec3f175\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 168])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 55f61404-8e20-4047-a779-ffe87d352f7e, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 68e120cd-095b-4827-9d1b-7b977952fe73, kind: Service { id: 55f61404-8e20-4047-a779-ffe87d352f7e }, name: Name(\"external-dns-55f61404-8e20-4047-a779-ffe87d352f7e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 7])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ec0c20b-4307-47e1-97b9-77d1ea3d29b3, kind: Service { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc }, name: Name(\"nexus-35af6b86-fd74-4104-b900-7a3edf8d03dc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 165, 129])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: d6d27046-4d61-4670-9a2f-1ffbb00db647, kind: Service { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599 }, name: Name(\"nexus-73e067cf-cfe8-480d-9d5d-fff7aeec0599\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 89])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 50145656-8d32-471a-8a33-3c47dc24d267, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8e49e841-d58f-44ba-beb8-68b0b0f8fc4c, kind: Service { id: 50145656-8d32-471a-8a33-3c47dc24d267 }, name: Name(\"nexus-50145656-8d32-471a-8a33-3c47dc24d267\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 470307fd-e881-437d-ad9f-f54f87acee44, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e05c51a-6602-4ef4-bc51-8d51552f353c, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 5f758d69-a06e-4648-b5fe-686e96c65857, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e996ca7-157f-4df1-ab30-04eac3863eb1, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: 524c932e-94cf-424b-b974-cd7c6d21eda9, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a330eff-2e60-4b6c-95a2-192e89df3181, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External } } } }, OmicronZoneConfig { id: bc50e194-c3e3-4fbb-a14d-26741c36b466, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.16552963Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.165990666Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.166012051Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.186985029Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.187016136Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.206072205Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.206100695Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.2241518Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.224183498Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.242114978Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.242143891Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.26097257Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.261007268Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.279169139Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.279212762Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.297216876Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.297247326Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.315149892Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.315176263Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55f61404-8e20-4047-a779-ffe87d352f7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.334063496Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.334102947Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55f61404-8e20-4047-a779-ffe87d352f7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.352422325Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.352454144Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.370512951Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.370547602Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.388601383Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.388630709Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.407169394Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.407199011Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 524c932e-94cf-424b-b974-cd7c6d21eda9, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.425292298Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.425345155Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 524c932e-94cf-424b-b974-cd7c6d21eda9, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.443366444Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.443398358Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a330eff-2e60-4b6c-95a2-192e89df3181, dataset_name: DatasetName { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.461732537Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.461774004Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a330eff-2e60-4b6c-95a2-192e89df3181, dataset_name: DatasetName { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc50e194-c3e3-4fbb-a14d-26741c36b466, dataset_name: DatasetName { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.481117116Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.48114467Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bc50e194-c3e3-4fbb-a14d-26741c36b466, dataset_name: DatasetName { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, dataset_name: DatasetName { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.499716264Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.499753962Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, dataset_name: DatasetName { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, dataset_name: DatasetName { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.517848795Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.517886924Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, dataset_name: DatasetName { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.536257806Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.536292748Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.536586888Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.536733432Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:16.628929034Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:16.628982849Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:16.628991446Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:16.628999161Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:16.629006596Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:16.629012746Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:16.629020275Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:16.629027634Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:16.629033606Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:16.629056088Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:16.629063944Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:16.629069738Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:16.629075663Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:16.629082619Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:16.629089641Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:16.62909686Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:16.629103002Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:16.629109798Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:16.629117017Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:16.629223379Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"c61365b6-916a-4eff-84bf-9c44babe5b96","remote_addr":"[fd00:1122:3344:101::1]:50078","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\", TimeNotSynchronized), (\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\", TimeNotSynchronized), (\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\", TimeNotSynchronized), (\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\", TimeNotSynchronized), (\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\", TimeNotSynchronized), (\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\", TimeNotSynchronized), (\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\", TimeNotSynchronized), (\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\", TimeNotSynchronized), (\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\", TimeNotSynchronized), (\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\", TimeNotSynchronized), (\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\", TimeNotSynchronized), (\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\", TimeNotSynchronized), (\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\", TimeNotSynchronized), (\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\", TimeNotSynchronized), (\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\", TimeNotSynchronized), (\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\", TimeNotSynchronized), (\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\", TimeNotSynchronized), (\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\", TimeNotSynchronized)]","latency_us":462306,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:16.629390418Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"373.708728ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c61365b6-916a-4eff-84bf-9c44babe5b96\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:50:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c61365b6-916a-4eff-84bf-9c44babe5b96\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6fc5bc0c-95b6-40a3-a5a9-955d74c48961, kind: Service { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee }, name: Name(\"ntp-f73788bd-d6b7-4d17-b028-bd04b6dd3aee\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: b46b9748-75d3-4248-95bb-b19942b724b7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: b605874a-c92a-4924-97a2-40d87990c249, kind: Service { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175 }, name: Name(\"external-dns-4ab1258a-9fc9-4e50-a387-e0cecec3f175\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 168])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 55f61404-8e20-4047-a779-ffe87d352f7e, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 68e120cd-095b-4827-9d1b-7b977952fe73, kind: Service { id: 55f61404-8e20-4047-a779-ffe87d352f7e }, name: Name(\"external-dns-55f61404-8e20-4047-a779-ffe87d352f7e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 7])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ec0c20b-4307-47e1-97b9-77d1ea3d29b3, kind: Service { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc }, name: Name(\"nexus-35af6b86-fd74-4104-b900-7a3edf8d03dc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 165, 129])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: d6d27046-4d61-4670-9a2f-1ffbb00db647, kind: Service { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599 }, name: Name(\"nexus-73e067cf-cfe8-480d-9d5d-fff7aeec0599\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 89])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 50145656-8d32-471a-8a33-3c47dc24d267, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8e49e841-d58f-44ba-beb8-68b0b0f8fc4c, kind: Service { id: 50145656-8d32-471a-8a33-3c47dc24d267 }, name: Name(\"nexus-50145656-8d32-471a-8a33-3c47dc24d267\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 470307fd-e881-437d-ad9f-f54f87acee44, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e05c51a-6602-4ef4-bc51-8d51552f353c, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 5f758d69-a06e-4648-b5fe-686e96c65857, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e996ca7-157f-4df1-ab30-04eac3863eb1, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: 524c932e-94cf-424b-b974-cd7c6d21eda9, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a330eff-2e60-4b6c-95a2-192e89df3181, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External } } } }, OmicronZoneConfig { id: bc50e194-c3e3-4fbb-a14d-26741c36b466, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.005993487Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.007078605Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.00709385Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.028046591Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.028079376Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.047556876Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.047588948Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.066276233Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.066305625Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.084290741Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.084322512Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.103016414Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.103080305Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.121207474Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.1212359Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.138992343Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.139018257Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.156896836Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.156936174Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55f61404-8e20-4047-a779-ffe87d352f7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.175081675Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.175112829Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55f61404-8e20-4047-a779-ffe87d352f7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.193420061Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.193453117Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.21197811Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.212013286Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.23015209Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.23019801Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.248049322Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.248080794Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 524c932e-94cf-424b-b974-cd7c6d21eda9, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.266207918Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.266243666Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 524c932e-94cf-424b-b974-cd7c6d21eda9, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.285330892Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.285365467Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a330eff-2e60-4b6c-95a2-192e89df3181, dataset_name: DatasetName { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.303969213Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.304001211Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a330eff-2e60-4b6c-95a2-192e89df3181, dataset_name: DatasetName { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc50e194-c3e3-4fbb-a14d-26741c36b466, dataset_name: DatasetName { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.322845684Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.322882848Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bc50e194-c3e3-4fbb-a14d-26741c36b466, dataset_name: DatasetName { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, dataset_name: DatasetName { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.341232845Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.341265283Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, dataset_name: DatasetName { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, dataset_name: DatasetName { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.359950945Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.360000512Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, dataset_name: DatasetName { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.378530325Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.378571782Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.378880286Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.379026896Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:17.473489752Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:17.473540162Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:17.473546462Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:17.473552556Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:17.473557825Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:17.473562653Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:17.473568673Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:17.473573623Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:17.473578441Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:17.473594585Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:17.473600482Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:17.473606032Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:17.473612013Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:17.473616824Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:17.473623874Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:17.473629789Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:17.473635517Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:17.47364132Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:17.473646206Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.4737213Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"ea6ef5b7-19d2-484e-a129-a3d997c69c58","remote_addr":"[fd00:1122:3344:101::1]:50078","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\", TimeNotSynchronized), (\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\", TimeNotSynchronized), (\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\", TimeNotSynchronized), (\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\", TimeNotSynchronized), (\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\", TimeNotSynchronized), (\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\", TimeNotSynchronized), (\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\", TimeNotSynchronized), (\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\", TimeNotSynchronized), (\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\", TimeNotSynchronized), (\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\", TimeNotSynchronized), (\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\", TimeNotSynchronized), (\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\", TimeNotSynchronized), (\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\", TimeNotSynchronized), (\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\", TimeNotSynchronized), (\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\", TimeNotSynchronized), (\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\", TimeNotSynchronized), (\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\", TimeNotSynchronized), (\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\", TimeNotSynchronized)]","latency_us":466219,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:17.473901615Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"356.458043ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ea6ef5b7-19d2-484e-a129-a3d997c69c58\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:50:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ea6ef5b7-19d2-484e-a129-a3d997c69c58\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6fc5bc0c-95b6-40a3-a5a9-955d74c48961, kind: Service { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee }, name: Name(\"ntp-f73788bd-d6b7-4d17-b028-bd04b6dd3aee\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: b46b9748-75d3-4248-95bb-b19942b724b7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: b605874a-c92a-4924-97a2-40d87990c249, kind: Service { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175 }, name: Name(\"external-dns-4ab1258a-9fc9-4e50-a387-e0cecec3f175\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 168])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 55f61404-8e20-4047-a779-ffe87d352f7e, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 68e120cd-095b-4827-9d1b-7b977952fe73, kind: Service { id: 55f61404-8e20-4047-a779-ffe87d352f7e }, name: Name(\"external-dns-55f61404-8e20-4047-a779-ffe87d352f7e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 7])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ec0c20b-4307-47e1-97b9-77d1ea3d29b3, kind: Service { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc }, name: Name(\"nexus-35af6b86-fd74-4104-b900-7a3edf8d03dc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 165, 129])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: d6d27046-4d61-4670-9a2f-1ffbb00db647, kind: Service { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599 }, name: Name(\"nexus-73e067cf-cfe8-480d-9d5d-fff7aeec0599\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 89])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 50145656-8d32-471a-8a33-3c47dc24d267, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8e49e841-d58f-44ba-beb8-68b0b0f8fc4c, kind: Service { id: 50145656-8d32-471a-8a33-3c47dc24d267 }, name: Name(\"nexus-50145656-8d32-471a-8a33-3c47dc24d267\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 470307fd-e881-437d-ad9f-f54f87acee44, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e05c51a-6602-4ef4-bc51-8d51552f353c, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 5f758d69-a06e-4648-b5fe-686e96c65857, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e996ca7-157f-4df1-ab30-04eac3863eb1, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: 524c932e-94cf-424b-b974-cd7c6d21eda9, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a330eff-2e60-4b6c-95a2-192e89df3181, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External } } } }, OmicronZoneConfig { id: bc50e194-c3e3-4fbb-a14d-26741c36b466, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.833136699Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.833415987Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.833428485Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.855195342Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.855230677Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.874363738Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.874396692Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.894928377Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.894956625Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.91301931Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.913055442Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.931572788Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.931600267Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.950899836Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.950930512Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.969755132Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.969784739Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.988836538Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:17.98886442Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55f61404-8e20-4047-a779-ffe87d352f7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.007064146Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.007093181Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55f61404-8e20-4047-a779-ffe87d352f7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.026159741Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.026185317Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.044931813Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.044969482Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.063746307Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.063781783Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.082237468Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.082268931Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 524c932e-94cf-424b-b974-cd7c6d21eda9, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.101259343Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.101297032Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 524c932e-94cf-424b-b974-cd7c6d21eda9, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.121868816Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.121896305Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a330eff-2e60-4b6c-95a2-192e89df3181, dataset_name: DatasetName { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.140376585Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.140402218Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a330eff-2e60-4b6c-95a2-192e89df3181, dataset_name: DatasetName { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc50e194-c3e3-4fbb-a14d-26741c36b466, dataset_name: DatasetName { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.159543913Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.159573004Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bc50e194-c3e3-4fbb-a14d-26741c36b466, dataset_name: DatasetName { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, dataset_name: DatasetName { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.177778189Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.177804851Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, dataset_name: DatasetName { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, dataset_name: DatasetName { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.195519298Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.19554703Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, dataset_name: DatasetName { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.213229524Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.213261438Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.213485684Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.213592637Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:18.308001688Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:18.308035702Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:18.308041692Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:18.308046867Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:18.308050964Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:18.308055877Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:18.308059992Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:18.308064811Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:18.308068965Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:18.308086965Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:18.308091221Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:18.308096049Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:18.308101131Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:18.308104994Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:18.30810895Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:18.308112888Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:18.308116919Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:18.308121653Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:18.308125572Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.30829681Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"a6624996-807d-4158-b347-846e3260fb98","remote_addr":"[fd00:1122:3344:101::1]:50078","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\", TimeNotSynchronized), (\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\", TimeNotSynchronized), (\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\", TimeNotSynchronized), (\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\", TimeNotSynchronized), (\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\", TimeNotSynchronized), (\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\", TimeNotSynchronized), (\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\", TimeNotSynchronized), (\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\", TimeNotSynchronized), (\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\", TimeNotSynchronized), (\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\", TimeNotSynchronized), (\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\", TimeNotSynchronized), (\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\", TimeNotSynchronized), (\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\", TimeNotSynchronized), (\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\", TimeNotSynchronized), (\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\", TimeNotSynchronized), (\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\", TimeNotSynchronized), (\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\", TimeNotSynchronized), (\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\", TimeNotSynchronized)]","latency_us":473728,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:18.308472042Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"658.307058ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a6624996-807d-4158-b347-846e3260fb98\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:50:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a6624996-807d-4158-b347-846e3260fb98\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6fc5bc0c-95b6-40a3-a5a9-955d74c48961, kind: Service { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee }, name: Name(\"ntp-f73788bd-d6b7-4d17-b028-bd04b6dd3aee\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: b46b9748-75d3-4248-95bb-b19942b724b7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: b605874a-c92a-4924-97a2-40d87990c249, kind: Service { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175 }, name: Name(\"external-dns-4ab1258a-9fc9-4e50-a387-e0cecec3f175\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 168])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 55f61404-8e20-4047-a779-ffe87d352f7e, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 68e120cd-095b-4827-9d1b-7b977952fe73, kind: Service { id: 55f61404-8e20-4047-a779-ffe87d352f7e }, name: Name(\"external-dns-55f61404-8e20-4047-a779-ffe87d352f7e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 7])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ec0c20b-4307-47e1-97b9-77d1ea3d29b3, kind: Service { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc }, name: Name(\"nexus-35af6b86-fd74-4104-b900-7a3edf8d03dc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 165, 129])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: d6d27046-4d61-4670-9a2f-1ffbb00db647, kind: Service { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599 }, name: Name(\"nexus-73e067cf-cfe8-480d-9d5d-fff7aeec0599\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 89])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 50145656-8d32-471a-8a33-3c47dc24d267, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8e49e841-d58f-44ba-beb8-68b0b0f8fc4c, kind: Service { id: 50145656-8d32-471a-8a33-3c47dc24d267 }, name: Name(\"nexus-50145656-8d32-471a-8a33-3c47dc24d267\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 470307fd-e881-437d-ad9f-f54f87acee44, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e05c51a-6602-4ef4-bc51-8d51552f353c, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 5f758d69-a06e-4648-b5fe-686e96c65857, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e996ca7-157f-4df1-ab30-04eac3863eb1, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: 524c932e-94cf-424b-b974-cd7c6d21eda9, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a330eff-2e60-4b6c-95a2-192e89df3181, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External } } } }, OmicronZoneConfig { id: bc50e194-c3e3-4fbb-a14d-26741c36b466, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.969782816Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.970403831Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.970432444Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.994595785Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:18.99463203Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.015115772Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.01523095Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.033580488Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.033622312Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.051058381Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.051098385Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.069713304Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.069770305Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.08778375Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.087813881Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.105547689Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.10558309Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.123527527Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.123559928Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55f61404-8e20-4047-a779-ffe87d352f7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.141777359Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.141832907Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55f61404-8e20-4047-a779-ffe87d352f7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.159843528Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.159884825Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.177576638Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.17767718Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.19697381Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.197007364Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.214659724Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.214689133Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 524c932e-94cf-424b-b974-cd7c6d21eda9, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.232634406Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.232678507Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 524c932e-94cf-424b-b974-cd7c6d21eda9, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.252350157Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.252381255Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a330eff-2e60-4b6c-95a2-192e89df3181, dataset_name: DatasetName { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.271704397Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.271727367Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a330eff-2e60-4b6c-95a2-192e89df3181, dataset_name: DatasetName { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc50e194-c3e3-4fbb-a14d-26741c36b466, dataset_name: DatasetName { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.290012842Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.290037302Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bc50e194-c3e3-4fbb-a14d-26741c36b466, dataset_name: DatasetName { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, dataset_name: DatasetName { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.308838178Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.308873569Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, dataset_name: DatasetName { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, dataset_name: DatasetName { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.327503879Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.327528348Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, dataset_name: DatasetName { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.34758191Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.347617086Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.347832482Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.34794852Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:19.438247863Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:19.438287791Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:19.43829516Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:19.438300204Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:19.438305032Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:19.43830986Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:19.438385072Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:19.438391327Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:19.438404864Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:19.438410648Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:19.438416339Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:19.43842112Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:19.438426755Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:19.438432586Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:19.438437284Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:19.438443181Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:19.438448027Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:19.438451656Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:19.438456334Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:19.438529957Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"2a493f28-ca3c-4341-9e9f-858d051f0976","remote_addr":"[fd00:1122:3344:101::1]:50078","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\", TimeNotSynchronized), (\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\", TimeNotSynchronized), (\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\", TimeNotSynchronized), (\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\", TimeNotSynchronized), (\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\", TimeNotSynchronized), (\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\", TimeNotSynchronized), (\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\", TimeNotSynchronized), (\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\", TimeNotSynchronized), (\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\", TimeNotSynchronized), (\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\", TimeNotSynchronized), (\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\", TimeNotSynchronized), (\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\", TimeNotSynchronized), (\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\", TimeNotSynchronized), (\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\", TimeNotSynchronized), (\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\", TimeNotSynchronized), (\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\", TimeNotSynchronized), (\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\", TimeNotSynchronized), (\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\", TimeNotSynchronized), (\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\", TimeNotSynchronized)]","latency_us":467175,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:19.438723743Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"629.28059ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2a493f28-ca3c-4341-9e9f-858d051f0976\", \"content-length\": \"124\", \"date\": \"Wed, 05 Jun 2024 19:50:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2a493f28-ca3c-4341-9e9f-858d051f0976\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6fc5bc0c-95b6-40a3-a5a9-955d74c48961, kind: Service { id: f73788bd-d6b7-4d17-b028-bd04b6dd3aee }, name: Name(\"ntp-f73788bd-d6b7-4d17-b028-bd04b6dd3aee\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: b46b9748-75d3-4248-95bb-b19942b724b7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: b605874a-c92a-4924-97a2-40d87990c249, kind: Service { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175 }, name: Name(\"external-dns-4ab1258a-9fc9-4e50-a387-e0cecec3f175\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 168])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 55f61404-8e20-4047-a779-ffe87d352f7e, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 68e120cd-095b-4827-9d1b-7b977952fe73, kind: Service { id: 55f61404-8e20-4047-a779-ffe87d352f7e }, name: Name(\"external-dns-55f61404-8e20-4047-a779-ffe87d352f7e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 7])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ec0c20b-4307-47e1-97b9-77d1ea3d29b3, kind: Service { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc }, name: Name(\"nexus-35af6b86-fd74-4104-b900-7a3edf8d03dc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 165, 129])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: d6d27046-4d61-4670-9a2f-1ffbb00db647, kind: Service { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599 }, name: Name(\"nexus-73e067cf-cfe8-480d-9d5d-fff7aeec0599\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 89])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 50145656-8d32-471a-8a33-3c47dc24d267, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8e49e841-d58f-44ba-beb8-68b0b0f8fc4c, kind: Service { id: 50145656-8d32-471a-8a33-3c47dc24d267 }, name: Name(\"nexus-50145656-8d32-471a-8a33-3c47dc24d267\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 470307fd-e881-437d-ad9f-f54f87acee44, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e05c51a-6602-4ef4-bc51-8d51552f353c, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 5f758d69-a06e-4648-b5fe-686e96c65857, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e996ca7-157f-4df1-ab30-04eac3863eb1, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External } } } }, OmicronZoneConfig { id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External } } } }, OmicronZoneConfig { id: 524c932e-94cf-424b-b974-cd7c6d21eda9, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External } } } }, OmicronZoneConfig { id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a330eff-2e60-4b6c-95a2-192e89df3181, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External } } } }, OmicronZoneConfig { id: bc50e194-c3e3-4fbb-a14d-26741c36b466, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.070977894Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.072206069Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.072250611Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05f414eb-1b07-4b76-9d7c-27a7db52ad4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.093266283Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.09329438Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46dff9fe-33a8-4272-a4b9-397eefbef96d, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.112366931Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.112404132Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d2b94d3-980a-43a1-8e8f-0973dee40a75, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.130580653Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.130617554Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb4b3cdb-0d2e-44b8-994b-0908726ddaeb, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.148564583Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.148588715Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d6f58b5e-cabb-496a-9b02-c2e904c1eb7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.167125622Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.167159363Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f6c2c6c4-927e-496d-873b-0e47ead3c40a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.185256675Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.185289066Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b46b9748-75d3-4248-95bb-b19942b724b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.20347592Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.203511545Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: be258c33-8299-48b4-94e9-f6c15d8dd8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.221572033Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.221602278Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55f61404-8e20-4047-a779-ffe87d352f7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.240722769Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.240751889Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55f61404-8e20-4047-a779-ffe87d352f7e, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.258918346Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.258961547Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2ace9c3-4702-4590-a5f2-70e552e5ff4e, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.277089431Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.277113123Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faeb0dce-f737-4f16-9e7f-1e9a288f9712, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c9449-467f-496f-ae4c-f2197a847844 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.29524849Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.295275283Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2bf92da-7b65-405c-8e55-ebbcd5c7fed9, dataset_name: DatasetName { pool_name: ZpoolName { id: d69c4ddc-54d3-4718-a115-7582c8252249 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.313298042Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.313334689Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cfb2e53c-9d55-4204-bc22-ea2aa48b7c80, dataset_name: DatasetName { pool_name: ZpoolName { id: 8e7826e7-938b-4b78-bd9b-c82573a27350 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 524c932e-94cf-424b-b974-cd7c6d21eda9, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.331329819Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.331377577Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 524c932e-94cf-424b-b974-cd7c6d21eda9, dataset_name: DatasetName { pool_name: ZpoolName { id: 87fce040-2c58-4185-8fd5-7dfb9378086a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.349297121Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.349325678Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4ac7506c-90ad-45f4-a164-7c6617f8d09a, dataset_name: DatasetName { pool_name: ZpoolName { id: 906c0a61-9cfc-4b51-be6b-6990f707b063 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a330eff-2e60-4b6c-95a2-192e89df3181, dataset_name: DatasetName { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.366932978Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.366961328Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a330eff-2e60-4b6c-95a2-192e89df3181, dataset_name: DatasetName { pool_name: ZpoolName { id: ca57da68-282d-4e81-9a3e-bfb822551ad8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc50e194-c3e3-4fbb-a14d-26741c36b466, dataset_name: DatasetName { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.384549011Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.384573893Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bc50e194-c3e3-4fbb-a14d-26741c36b466, dataset_name: DatasetName { pool_name: ZpoolName { id: df264fcd-b75d-4ee3-9b2e-e281836815f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, dataset_name: DatasetName { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.402425296Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.402461241Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3d9237cb-2717-48f0-8bd3-23d4bf504717, dataset_name: DatasetName { pool_name: ZpoolName { id: 9fef4b9e-a443-49fc-b47e-d8cdc87127ad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, dataset_name: DatasetName { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.420506558Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.420544987Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ff0fc90-2090-4f92-b269-bbfcff3f2db6, dataset_name: DatasetName { pool_name: ZpoolName { id: c495d42b-321e-4420-8635-102c37b30fb2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.438163294Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.438199736Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.438402578Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.438524775Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.524761972Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.550462707Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.550660374Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.550685668Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.565770542Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.566011627Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.566044347Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.600079733Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.600121331Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.600139593Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.741999349Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.742216817Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.80040801Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.800619252Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.800638753Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.800684607Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.800730236Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.80077264Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.800778359Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.800977442Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.805526806Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.810440558Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.814950863Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.820430531Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.825765189Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.830815732Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.969477043Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.973874374Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.978498626Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.982393015Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.986910873Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.991633341Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:20.99673925Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.001551538Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.007097752Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.012535332Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.01884896Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.027497034Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.033262839Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.03862924Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.045459391Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.050910357Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.056424104Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.062879947Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.069249589Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.075218351Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.082035Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.154096336Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.234579541Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.291665273Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.349838572Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.408162298Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.413581601Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.413595026Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.500809468Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 192, 7]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte1\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.500875282Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.503669926Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.503686557Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.607568147Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 180, 184]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte2\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.607628927Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.612150415Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.612165293Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.727080801Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 138, 89]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte3\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.727133753Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.730395421Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.730411124Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.82252316Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 165, 129]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte4\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.822570918Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.824973976Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.824990487Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.931034243Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 201, 168]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte5\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.931101116Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.937040391Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.937057699Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.937379527Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.937392896Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.937401418Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.937409509Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.937480769Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.937489302Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.937601139Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.937608076Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.953096923Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:21.955683029Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:22.034782716Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:22.039920476Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:22.043565754Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:22.043583998Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:22.043602749Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:22.046812246Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:22.049808324Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:22.049822836Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:22.184279106Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:22.190370635Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:22.196990178Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:22.203693066Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:22.210211149Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:22.218541454Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:22.226039505Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:22.232036468Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:22.238048747Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:22.245104389Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:24.798508524Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:24.799171117Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:24.799227096Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:24.810461471Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:24.821929882Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:24.822252538Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:25.492148628Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:25.492513212Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:25.502549343Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:25.503532529Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:25.504267257Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:25.505357768Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:25.505455739Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:25.50607705Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:25.508656219Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:25.508756722Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:25.508806598Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:25.517820589Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:25.526305935Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:25.534907926Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:25.543095361Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:25.550396716Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:25.557632972Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:25.759827595Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:25.830759095Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:25.831342278Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:28.168555718Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"1.514173246s"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 66.404954ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:28.281905753Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 37.698712ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:28.296312311Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\") failed: Property not found. retry in 62.458996ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:28.308522916Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\") failed: Property not found. retry in 74.456365ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:28.332498131Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 27.807376ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:28.37155454Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\") failed: Property not found. retry in 50.604378ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:28.431601346Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\") failed: Property not found. retry in 57.459568ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:28.580484569Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\") failed: Property not found. retry in 69.656038ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:28.779213996Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\") failed: Property not found. retry in 28.511758ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:28.877349209Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\") failed: Property not found. retry in 25.774436ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:28.94349633Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 132.585923ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.00072621Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\") failed: Property not found. retry in 61.929024ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.046126646Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\") failed: Property not found. retry in 135.412786ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.108393422Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\") failed: Property not found. retry in 31.990661ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.401576918Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\") failed: Property not found. retry in 44.743851ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.566419833Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 96.817216ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.697989602Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\") failed: Property not found. retry in 142.779365ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.746383209Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\") failed: Property not found. retry in 78.503296ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.7604Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\") failed: Property not found. retry in 91.679475ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.776204842Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 65.299142ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.79233144Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 98.405674ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.806420076Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\") failed: Property not found. retry in 72.053081ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.822746264Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\") failed: Property not found. retry in 92.571855ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.842514467Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 115.638868ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.857528498Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\") failed: Property not found. retry in 120.947514ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.872305539Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\") failed: Property not found. retry in 296.052958ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.886758764Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\") failed: Property not found. retry in 130.304223ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.897497164Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\") failed: Property not found. retry in 116.513092ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.908116687Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 158.127285ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.917573021Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\") failed: Property not found. retry in 115.266293ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.928045525Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\") failed: Property not found. retry in 142.12331ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.938664325Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\") failed: Property not found. retry in 214.668667ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.94894658Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 114.811434ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.959339729Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 209.597567ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.970547434Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\") failed: Property not found. retry in 104.664617ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.983811304Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\") failed: Property not found. retry in 267.219009ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:29.995650312Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 497.640661ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:30.006826916Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\") failed: Property not found. retry in 536.659129ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:30.178073357Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\") failed: Property not found. retry in 294.333741ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:30.20533984Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\") failed: Property not found. retry in 125.285093ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:30.229610946Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 233.456955ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:30.259733313Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\") failed: Property not found. retry in 327.859719ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:30.285388243Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\") failed: Property not found. retry in 234.367456ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:30.324438982Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\") failed: Property not found. retry in 281.813617ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:30.355053766Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 213.817605ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:30.457331947Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 338.081073ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:30.50670634Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\") failed: Property not found. retry in 246.994449ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:30.580656225Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\") failed: Property not found. retry in 494.571136ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:30.6687496Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 401.472997ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:30.748689179Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\") failed: Property not found. retry in 1.181485428s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:30.780986199Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\") failed: Property not found. retry in 319.578581ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:30.839929635Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\") failed: Property not found. retry in 411.883831ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:30.859818094Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\") failed: Property not found. retry in 508.658819ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:30.883811387Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 1.029988797s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:30.904728138Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\") failed: Property not found. retry in 852.892933ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:30.924829914Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\") failed: Property not found. retry in 647.610906ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:30.940385924Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\") failed: Property not found. retry in 829.461394ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:30.957583237Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 506.07518ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:30.978433868Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 836.379208ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:31.000086224Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\") failed: Property not found. retry in 692.981449ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:31.017045769Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 733.458851ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:31.168095785Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\") failed: Property not found. retry in 879.752759ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:31.182555823Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\") failed: Property not found. retry in 889.822858ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:31.198361633Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\") failed: Property not found. retry in 916.154955ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:31.304276886Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\") failed: Property not found. retry in 747.391561ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:31.408690336Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 604.901474ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:31.505938648Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\") failed: Property not found. retry in 897.531906ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:31.609292847Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\") failed: Property not found. retry in 811.06633ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:31.726438694Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\") failed: Property not found. retry in 843.182397ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:31.793935817Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\") failed: Property not found. retry in 1.206834369s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:31.807731065Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 592.661946ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:31.854538694Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 525.017166ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:31.962156198Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\") failed: Property not found. retry in 765.632185ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:32.04521181Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 859.169018ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:32.062515082Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\") failed: Property not found. retry in 1.328407347s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:32.080318824Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\") failed: Property not found. retry in 1.190839997s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:32.106860789Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 673.338268ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:32.130579901Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\") failed: Property not found. retry in 1.298171153s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:32.172803713Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:32.18839551Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:32.18939006Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:32.197050578Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:32.197633618Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\") failed: Property not found. retry in 674.009584ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:32.239884307Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 1.041892551s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:32.461179186Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\") failed: Property not found. retry in 65.82624ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:32.473531066Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\") failed: Property not found. retry in 67.093472ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:32.483874366Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 527.873813ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:32.503558499Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\") failed: Property not found. retry in 550.811718ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:32.522846292Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\") failed: Property not found. retry in 93.733675ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:32.554882138Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\") failed: Property not found. retry in 902.762753ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:32.567458383Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\") failed: Property not found. retry in 93.614173ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:32.577925736Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\") failed: Property not found. retry in 630.317809ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:32.65200291Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\") failed: Property not found. retry in 163.648061ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:32.666272578Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\") failed: Property not found. retry in 189.046617ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:32.684210503Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 1.460940087s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:32.81974775Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\") failed: Property not found. retry in 584.443433ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:32.833376728Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\") failed: Property not found. retry in 395.681025ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:32.844262065Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\") failed: Property not found. retry in 257.724778ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:32.886393191Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\") failed: Property not found. retry in 1.324294318s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:32.928008502Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 1.434334793s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:32.94004043Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:32.953731445Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:32.953753627Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:32.954025209Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:32.961172775Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\") failed: Property not found. retry in 1.055240226s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.031105416Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:33.032532011Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:33.032630911Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 770.827236ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.049996189Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\") failed: Property not found. retry in 1.439772071s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.088682249Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\") failed: Property not found. retry in 624.902674ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.158221895Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\") failed: Property not found. retry in 31.188319ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.168513433Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\") failed: Property not found. retry in 51.545558ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.17727685Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:33.178444806Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48658"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:33.189496954Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"4f094d5d-fcb1-4dca-82fb-085aac6bf4e8","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48658","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1473,"response_code":"200"} {"msg":"wait for service Some(\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\") failed: Property not found. retry in 121.743767ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.208531115Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\") failed: Property not found. retry in 30.833619ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.221188439Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\") failed: Property not found. retry in 77.045001ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.240348451Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\") failed: Property not found. retry in 613.551139ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.251945821Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\") failed: Property not found. retry in 94.267119ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.261600881Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\") failed: Property not found. retry in 1.468178812s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.297926782Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\") failed: Property not found. retry in 1.282117575s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.313782175Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\") failed: Property not found. retry in 101.510586ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.437647942Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\") failed: Property not found. retry in 230.075912ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.610916756Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\") failed: Property not found. retry in 1.334593891s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.628388374Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\") failed: Property not found. retry in 1.13564367s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.643172195Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\") failed: Property not found. retry in 1.113377216s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.65696219Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 779.281021ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.670974736Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\") failed: Property not found. retry in 1.421271904s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.685181874Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\") failed: Property not found. retry in 188.521942ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.70034561Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\") failed: Property not found. retry in 256.308763ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.713643068Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\") failed: Property not found. retry in 955.503933ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.799517071Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 868.311803ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.83417732Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\") failed: Property not found. retry in 577.542087ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.852557356Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\") failed: Property not found. retry in 683.979508ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.880850149Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\") failed: Property not found. retry in 309.738041ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.901974812Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\") failed: Property not found. retry in 1.03689467s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:33.986833558Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\") failed: Property not found. retry in 1.408600865s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:34.105573448Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\") failed: Property not found. retry in 478.181143ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:34.232596097Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\") failed: Property not found. retry in 1.494355207s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:34.270307711Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 1.260237312s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:34.296679414Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 1.459881977s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:34.391496237Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\") failed: Property not found. retry in 454.904494ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:34.444505738Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 924.270722ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:34.463472614Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\") failed: Property not found. retry in 1.466878675s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:34.544214365Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\") failed: Property not found. retry in 1.200671126s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:34.57662637Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\") failed: Property not found. retry in 1.210410067s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:34.611858797Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 939.204235ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:34.718399717Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\") failed: Property not found. retry in 1.416623097s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:34.730255082Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\") failed: Property not found. retry in 1.033708168s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:34.770237349Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\") failed: Property not found. retry in 585.797212ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:34.782694195Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\") failed: Property not found. retry in 1.49687788s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:34.793171815Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\") failed: Property not found. retry in 1.262094523s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:34.804449142Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\") failed: Property not found. retry in 1.020153848s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:34.914124547Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\") failed: Property not found. retry in 1.469261999s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:34.979149803Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\") failed: Property not found. retry in 1.210739036s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:35.040904099Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\") failed: Property not found. retry in 1.175226056s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:35.124106435Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\") failed: Property not found. retry in 1.050683667s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:35.384361832Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 583.708115ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:35.403363597Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\") failed: Property not found. retry in 1.146497952s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:35.531143094Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 1.186190937s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:35.573322951Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 893.162826ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:35.673295494Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\") failed: Property not found. retry in 1.348257156s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:35.782860846Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\") failed: Property not found. retry in 1.250067645s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:35.793665061Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\") failed: Property not found. retry in 862.446584ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:35.817779581Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\") failed: Property not found. retry in 1.083305463s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:35.835603455Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 1.107394005s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:35.866130261Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\") failed: Property not found. retry in 1.475463692s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:35.948986586Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 1.211822038s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:36.000190804Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\") failed: Property not found. retry in 1.377546533s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:36.031178551Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\") failed: Property not found. retry in 797.233939ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:36.083915752Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\") failed: Property not found. retry in 755.279652ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:36.163943906Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\") failed: Property not found. retry in 690.399359ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:36.26936396Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\") failed: Property not found. retry in 1.446427364s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:36.305962859Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\") failed: Property not found. retry in 1.221283671s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:36.314172553Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\") failed: Property not found. retry in 1.305400129s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:36.447101095Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\") failed: Property not found. retry in 1.161532985s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:36.460923691Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 1.057594457s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:36.578299204Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\") failed: Property not found. retry in 589.349198ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:36.689203153Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\") failed: Property not found. retry in 857.815172ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:36.696704263Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 814.373575ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:36.771427324Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\") failed: Property not found. retry in 635.18046ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:36.892449381Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\") failed: Property not found. retry in 1.179611297s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:36.929196222Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\") failed: Property not found. retry in 870.611555ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:36.936732068Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\") failed: Property not found. retry in 672.641927ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:36.970577931Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 939.253939ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:36.995579947Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\") failed: Property not found. retry in 1.462979955s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:37.057027801Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\") failed: Property not found. retry in 1.410664853s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:37.143924613Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 511.871516ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:37.273666705Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\") failed: Property not found. retry in 1.374392401s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:37.320071145Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\") failed: Property not found. retry in 682.172806ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:37.450890962Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\") failed: Property not found. retry in 1.381790233s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:37.460142284Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\") failed: Property not found. retry in 552.629637ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:37.540199Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\") failed: Property not found. retry in 1.016281429s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:37.548690281Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\") failed: Property not found. retry in 714.626533ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:37.566283549Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 940.196045ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:37.598325216Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 1.326021665s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:37.654512013Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\") failed: Property not found. retry in 856.718478ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:37.662348632Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\") failed: Property not found. retry in 932.632829ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:37.670939677Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:37.673826901Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:37.674040901Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\") failed: Property not found. retry in 657.836134ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:37.766164684Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\") failed: Property not found. retry in 1.188287218s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:37.77444647Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 1.133401967s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:37.796642327Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\") failed: Property not found. retry in 1.429418641s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:37.819145425Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\") failed: Property not found. retry in 33.366809ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:37.867897318Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\") failed: Property not found. retry in 68.037508ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:37.910025976Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 760.911118ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:37.947706783Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\") failed: Property not found. retry in 194.237061ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:37.988676369Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\") failed: Property not found. retry in 1.438700794s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:38.109074812Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\") failed: Property not found. retry in 976.63251ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:38.12549312Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\") failed: Property not found. retry in 1.083371595s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:38.147000804Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\") failed: Property not found. retry in 209.701565ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:38.192639152Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\") failed: Property not found. retry in 733.394379ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:38.293090294Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\") failed: Property not found. retry in 736.000578ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:38.413363744Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\") failed: Property not found. retry in 1.41580428s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:38.435522654Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\") failed: Property not found. retry in 907.9546ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:38.530728373Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\") failed: Property not found. retry in 1.346034526s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:38.539407852Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 538.203293ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:38.55120338Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\") failed: Property not found. retry in 885.205681ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:38.567204433Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\") failed: Property not found. retry in 1.437661735s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:38.575902967Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\") failed: Property not found. retry in 516.731116ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:38.615934995Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\") failed: Property not found. retry in 712.743535ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:38.707725421Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 955.559854ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:38.719749957Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\") failed: Property not found. retry in 1.498407217s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:38.869925764Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 668.015412ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:38.947120735Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\") failed: Property not found. retry in 1.144465835s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:38.981743235Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 631.333758ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:38.999360586Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\") failed: Property not found. retry in 536.542824ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:39.043291676Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 764.772511ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:39.10158865Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\") failed: Property not found. retry in 1.178123077s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:39.115337279Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\") failed: Property not found. retry in 615.809347ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:39.144460042Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\") failed: Property not found. retry in 766.421935ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:39.16022826Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\") failed: Property not found. retry in 848.533598ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:39.244323118Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\") failed: Property not found. retry in 969.743256ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:39.262360881Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\") failed: Property not found. retry in 1.220590682s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:39.432869202Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\") failed: Property not found. retry in 1.097395308s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:39.451532229Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\") failed: Property not found. retry in 1.335991601s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:39.464937742Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\") failed: Property not found. retry in 885.609886ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:39.562982992Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\") failed: Property not found. retry in 1.479568805s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:39.589617898Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 1.251400931s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:39.626616555Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 782.80649ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:39.641546036Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 625.596415ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:39.686740336Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\") failed: Property not found. retry in 1.198313411s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:39.771535277Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\") failed: Property not found. retry in 1.454689937s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:39.864289529Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 1.378701027s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:39.877267715Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\") failed: Property not found. retry in 639.560356ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:39.898516194Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\") failed: Property not found. retry in 680.441947ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:39.939401718Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\") failed: Property not found. retry in 580.470226ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:40.026686466Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\") failed: Property not found. retry in 1.305909059s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:40.104463872Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\") failed: Property not found. retry in 711.180498ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:40.136819576Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\") failed: Property not found. retry in 925.641163ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:40.242648194Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\") failed: Property not found. retry in 1.096859375s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:40.304448913Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 1.279444992s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:40.322266493Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\") failed: Property not found. retry in 1.144021365s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:40.378440909Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 1.394980504s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:40.434582695Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\") failed: Property not found. retry in 1.302142747s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:40.459013782Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\") failed: Property not found. retry in 849.85169ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:40.546844035Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\") failed: Property not found. retry in 802.633935ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:40.558119091Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\") failed: Property not found. retry in 1.222412917s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:40.616069179Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\") failed: Property not found. retry in 810.992377ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:40.628323588Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\") failed: Property not found. retry in 1.07251668s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:40.662739911Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\") failed: Property not found. retry in 1.26507911s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:40.812965958Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\") failed: Property not found. retry in 1.208457862s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:40.857305874Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 1.084371006s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:40.889423292Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\") failed: Property not found. retry in 554.204643ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:40.980835681Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\") failed: Property not found. retry in 1.384926507s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:41.080396271Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\") failed: Property not found. retry in 659.63664ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:41.178496917Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 1.041966187s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:41.266066264Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\") failed: Property not found. retry in 692.636063ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:41.328375945Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\") failed: Property not found. retry in 1.411050666s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:41.371142624Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\") failed: Property not found. retry in 922.119545ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:41.405238389Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\") failed: Property not found. retry in 1.033932329s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:41.413341871Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\") failed: Property not found. retry in 668.141839ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:41.421343049Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\") failed: Property not found. retry in 1.42579492s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:41.447839995Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\") failed: Property not found. retry in 1.052662965s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:41.530923216Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\") failed: Property not found. retry in 748.11835ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:41.544131408Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 1.028774825s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:41.610808712Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\") failed: Property not found. retry in 1.491601036s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:41.745148074Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\") failed: Property not found. retry in 1.402387235s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:41.770743518Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 1.064539665s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:41.838959179Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\") failed: Property not found. retry in 506.400503ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:41.846897026Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\") failed: Property not found. retry in 1.243343652s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:41.855092989Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 914.062744ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:41.982705872Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\") failed: Property not found. retry in 533.892284ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:42.03089549Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\") failed: Property not found. retry in 1.379527813s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:42.07875921Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\") failed: Property not found. retry in 1.073169495s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:42.089529764Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\") failed: Property not found. retry in 1.216620965s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:42.09909314Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\") failed: Property not found. retry in 977.290812ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:42.301846416Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 589.909776ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:42.316761628Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\") failed: Property not found. retry in 1.179754082s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:42.337501398Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\") failed: Property not found. retry in 890.757985ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:42.364119546Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\") failed: Property not found. retry in 1.074394668s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:42.45704417Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\") failed: Property not found. retry in 883.302668ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:42.474086182Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\") failed: Property not found. retry in 573.670012ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:42.575562468Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\") failed: Property not found. retry in 1.412580564s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:42.593066871Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 1.06498698s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:42.649521454Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\") failed: Property not found. retry in 641.587492ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:42.793649406Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\") failed: Property not found. retry in 667.874175ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:42.884613355Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 562.581429ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:42.907539165Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 932.010203ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:42.915462002Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 974.999384ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:42.92342978Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\") failed: Property not found. retry in 1.496983146s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:43.108339675Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\") failed: Property not found. retry in 999.073787ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:43.15810865Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\") failed: Property not found. retry in 1.484967689s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:43.171258181Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\") failed: Property not found. retry in 536.046968ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:43.181538974Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\") failed: Property not found. retry in 842.702694ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:43.247712076Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\") failed: Property not found. retry in 1.390555634s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:43.264430971Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\") failed: Property not found. retry in 955.686946ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:43.289357639Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\") failed: Property not found. retry in 1.386132108s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:43.325121663Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\") failed: Property not found. retry in 973.988549ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:43.367185265Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\") failed: Property not found. retry in 1.28659392s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:43.446277773Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\") failed: Property not found. retry in 530.119856ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:43.468821625Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 826.351643ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:43.480238594Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\") failed: Property not found. retry in 1.211330525s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:43.526829164Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\") failed: Property not found. retry in 551.667978ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:43.541533512Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\") failed: Property not found. retry in 877.654592ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:43.562462286Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 692.474801ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:43.724359458Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\") failed: Property not found. retry in 1.331859035s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:43.732535351Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 1.185145501s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:43.857743746Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 666.392732ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:43.911102706Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\") failed: Property not found. retry in 1.154339655s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:44.008547698Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\") failed: Property not found. retry in 1.199808802s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:44.016801972Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\") failed: Property not found. retry in 1.480571046s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:44.100915837Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\") failed: Property not found. retry in 520.315124ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:44.108861075Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\") failed: Property not found. retry in 1.279830027s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:44.166950376Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\") failed: Property not found. retry in 1.019794629s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:44.254673177Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 886.563263ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:44.316105337Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\") failed: Property not found. retry in 891.623514ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:44.352386823Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 581.741522ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:44.428436689Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\") failed: Property not found. retry in 1.473936069s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:44.451130025Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 1.103247087s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:44.58985951Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\") failed: Property not found. retry in 862.488096ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:44.619665033Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\") failed: Property not found. retry in 1.025028494s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:44.643292661Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\") failed: Property not found. retry in 753.075858ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:44.669441448Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\") failed: Property not found. retry in 1.302999418s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:44.681665207Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\") failed: Property not found. retry in 1.245625908s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:44.726458614Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\") failed: Property not found. retry in 1.256250169s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:44.74784963Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\") failed: Property not found. retry in 939.765916ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:44.761102572Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:44.899618682Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"4.584534278s"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 781.60377ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:45.02448728Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 1.142639075s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:45.0554048Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\") failed: Property not found. retry in 1.325864017s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:45.079499119Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\") failed: Property not found. retry in 1.05888474s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:45.175568507Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 786.720494ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:45.215683614Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\") failed: Property not found. retry in 1.123685583s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:45.230442692Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\") failed: Property not found. retry in 1.123086995s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:45.257287896Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\") failed: Property not found. retry in 876.801926ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:45.300701271Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\") failed: Property not found. retry in 738.989373ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:45.449232247Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\") failed: Property not found. retry in 604.720231ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:45.470699944Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\") failed: Property not found. retry in 712.547826ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:45.499574786Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\") failed: Property not found. retry in 891.608803ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:45.605264451Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\") failed: Property not found. retry in 729.881701ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:45.708532099Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\") failed: Property not found. retry in 915.824869ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:45.724441605Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 1.003955948s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:45.739047159Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 890.639024ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:45.828682445Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\") failed: Property not found. retry in 1.441161291s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:45.941768955Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\") failed: Property not found. retry in 1.437193511s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:45.993272131Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\") failed: Property not found. retry in 1.486400915s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:46.011631474Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 843.298015ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:46.03262536Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\") failed: Property not found. retry in 659.403738ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:46.053002971Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\") failed: Property not found. retry in 536.694855ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:46.094240164Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\") failed: Property not found. retry in 958.023115ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:46.192633197Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\") failed: Property not found. retry in 1.188230749s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:46.204919677Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 719.220756ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:46.216908671Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\") failed: Property not found. retry in 1.325692736s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:46.229125609Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\") failed: Property not found. retry in 1.021344412s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:46.244633155Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\") failed: Property not found. retry in 1.386807158s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:46.369167304Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\") failed: Property not found. retry in 641.382655ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:46.393897648Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\") failed: Property not found. retry in 637.696019ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:46.421476999Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\") failed: Property not found. retry in 679.694869ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:46.453597207Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\") failed: Property not found. retry in 1.455113587s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:46.511338401Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\") failed: Property not found. retry in 964.748788ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:46.64346624Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\") failed: Property not found. retry in 1.308237793s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:46.656065507Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\") failed: Property not found. retry in 724.094429ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:46.724228847Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 1.308599012s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:46.735899272Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 1.322153405s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:46.754665515Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 970.468374ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:46.888546796Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 913.564872ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:46.950711263Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\") failed: Property not found. retry in 875.675516ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:47.052278311Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\") failed: Property not found. retry in 1.013469529s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:47.072092629Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\") failed: Property not found. retry in 1.210185052s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:47.145672796Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\") failed: Property not found. retry in 1.221808921s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:47.162412251Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\") failed: Property not found. retry in 1.458573023s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:47.280056148Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\") failed: Property not found. retry in 542.647631ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:47.395293358Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\") failed: Property not found. retry in 892.996648ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:47.408109291Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\") failed: Property not found. retry in 1.092333555s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:47.441844657Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\") failed: Property not found. retry in 1.45529066s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:47.461306847Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\") failed: Property not found. retry in 650.574707ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:47.51099484Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\") failed: Property not found. retry in 1.173127241s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:47.569926841Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\") failed: Property not found. retry in 751.058865ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:47.622457251Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\") failed: Property not found. retry in 635.025055ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:47.769895702Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 620.063069ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:47.872514124Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 810.587442ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:47.884864656Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\") failed: Property not found. retry in 794.488462ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:47.944270458Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\") failed: Property not found. retry in 1.245408899s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:47.958229488Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\") failed: Property not found. retry in 1.093338675s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:47.978317252Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\") failed: Property not found. retry in 1.211503497s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:47.993330887Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 545.132624ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:48.060024591Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 674.455278ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:48.091086244Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\") failed: Property not found. retry in 753.887292ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:48.104584113Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\") failed: Property not found. retry in 1.125749197s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:48.177694609Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\") failed: Property not found. retry in 1.3488743s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:48.316505916Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\") failed: Property not found. retry in 806.090809ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:48.374467474Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857\") failed: Property not found. retry in 1.25746095s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:48.388669129Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\") failed: Property not found. retry in 1.442272761s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:48.402071196Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\") failed: Property not found. retry in 1.306075631s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:48.417694205Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1\") failed: Property not found. retry in 1.49205342s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:48.509376469Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\") failed: Property not found. retry in 623.232675ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:48.549400684Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 1.093926909s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:48.620904602Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 1.459999087s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:48.718087006Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\") failed: Property not found. retry in 1.378301921s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:48.752669533Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\") failed: Property not found. retry in 1.061583498s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:48.768566688Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 752.253873ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:48.786816899Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\") failed: Property not found. retry in 1.148581514s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:48.803715898Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\") failed: Property not found. retry in 808.981405ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:48.876286651Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\") failed: Property not found. retry in 815.268836ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:48.935487469Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\") failed: Property not found. retry in 563.191176ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:49.089807242Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\") failed: Property not found. retry in 672.962298ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:49.19141596Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\") failed: Property not found. retry in 573.054761ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:49.206572098Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\") failed: Property not found. retry in 1.031995169s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:49.218765416Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\") failed: Property not found. retry in 1.342265265s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:49.232728846Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\") failed: Property not found. retry in 1.072782864s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:49.324235566Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 1.236730305s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:49.553214481Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\") failed: Property not found. retry in 691.218185ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:49.683982088Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\") failed: Property not found. retry in 1.317212734s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:49.700116949Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\") failed: Property not found. retry in 1.032298393s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:49.71539824Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 536.92689ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:49.729991774Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\") failed: Property not found. retry in 927.565255ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:49.744649551Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\") failed: Property not found. retry in 696.019743ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:49.766175979Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\") failed: Property not found. retry in 624.550736ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:49.794005776Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e\") failed: Property not found. retry in 1.292065735s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:49.841657951Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\") failed: Property not found. retry in 688.140878ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:49.861287284Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\") failed: Property not found. retry in 1.058205836s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:49.881170394Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\") failed: Property not found. retry in 954.636018ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:49.970723218Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\") failed: Property not found. retry in 1.113305748s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:50.140998857Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 1.066551243s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:50.190312442Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e\") failed: Property not found. retry in 1.358669374s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:50.26268626Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 875.30968ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:50.278605536Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712\") failed: Property not found. retry in 1.259141952s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:50.38510549Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44\") failed: Property not found. retry in 1.268840352s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:50.408188175Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\") failed: Property not found. retry in 748.938709ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:50.429362435Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c\") failed: Property not found. retry in 1.062130301s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:50.474958391Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599\") failed: Property not found. retry in 1.347071358s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:50.562259122Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466\") failed: Property not found. retry in 1.456146937s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:50.583870052Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc\") failed: Property not found. retry in 1.068882175s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:50.685230085Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80\") failed: Property not found. retry in 989.392951ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:50.758965396Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717\") failed: Property not found. retry in 698.119447ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:50.800692931Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6\") failed: Property not found. retry in 1.307962186s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:50.936233691Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181\") failed: Property not found. retry in 699.354326ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:50.948853959Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175\") failed: Property not found. retry in 668.90393ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:51.028171111Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9\") failed: Property not found. retry in 1.354999171s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:51.164779436Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267\") failed: Property not found. retry in 787.032312ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:51.18779332Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a\") failed: Property not found. retry in 1.434095768s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:51.264668575Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9\") failed: Property not found. retry in 1.398000232s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:51.272340436Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.737357651Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_nexus_73e067cf-cfe8-480d-9d5d-fff7aeec0599"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.737415942Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_crucible_faeb0dce-f737-4f16-9e7f-1e9a288f9712"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.737423002Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_crucible_c2bf92da-7b65-405c-8e55-ebbcd5c7fed9"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.737428782Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_nexus_35af6b86-fd74-4104-b900-7a3edf8d03dc"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.737434592Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_crucible_pantry_2e05c51a-6602-4ef4-bc51-8d51552f353c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.737440653Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_crucible_pantry_5f758d69-a06e-4648-b5fe-686e96c65857"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.737446403Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_external_dns_4ab1258a-9fc9-4e50-a387-e0cecec3f175"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.737452223Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_oximeter_470307fd-e881-437d-ad9f-f54f87acee44"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.737457773Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_crucible_cfb2e53c-9d55-4204-bc22-ea2aa48b7c80"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.737472503Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_crucible_524c932e-94cf-424b-b974-cd7c6d21eda9"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.737478724Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_crucible_4ac7506c-90ad-45f4-a164-7c6617f8d09a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.737484324Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_external_dns_55f61404-8e20-4047-a779-ffe87d352f7e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.737489894Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_clickhouse_d2ace9c3-4702-4590-a5f2-70e552e5ff4e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.737495544Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_crucible_3a330eff-2e60-4b6c-95a2-192e89df3181"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.737500914Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.737506004Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_crucible_3d9237cb-2717-48f0-8bd3-23d4bf504717"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.737511594Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_crucible_pantry_0e996ca7-157f-4df1-ab30-04eac3863eb1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.737628987Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_crucible_6ff0fc90-2090-4f92-b269-bbfcff3f2db6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.737637617Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_nexus_50145656-8d32-471a-8a33-3c47dc24d267"} {"msg":"Writing ledger to /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.743307374Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.754350782Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.764204404Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"7b33eb60-6aba-45c6-aa85-ffb640290919","remote_addr":"[fd00:1122:3344:101::1]:50078","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":32638194,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.774967478Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1237"} {"msg":"Writing ledger to /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.775305866Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.775671585Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.788445787Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:681"} {"msg":"Nexus address: [fd00:1122:3344:101::a]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.795071563Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:694"} {"msg":"rack_network_config: RackNetworkConfigV1 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV1 {\n addresses: [\n V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:52.850495612Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:818"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2c4cdae1-0bef-46b7-80a3-3be5f1adb047/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:53.998882032Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:50:54.000648193Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)\"","delay":"9.201000277s"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2c4cdae1-0bef-46b7-80a3-3be5f1adb047/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:54.074033026Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2c4cdae1-0bef-46b7-80a3-3be5f1adb047/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:54.245959595Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2c4cdae1-0bef-46b7-80a3-3be5f1adb047/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:54.356418632Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2c4cdae1-0bef-46b7-80a3-3be5f1adb047/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:54.579852567Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2c4cdae1-0bef-46b7-80a3-3be5f1adb047/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:54.863292818Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2c4cdae1-0bef-46b7-80a3-3be5f1adb047/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:55.056658133Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2c4cdae1-0bef-46b7-80a3-3be5f1adb047/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:55.286255623Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2c4cdae1-0bef-46b7-80a3-3be5f1adb047/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:55.795875908Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2c4cdae1-0bef-46b7-80a3-3be5f1adb047/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:56.013312577Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2c4cdae1-0bef-46b7-80a3-3be5f1adb047/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:56.446838772Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2c4cdae1-0bef-46b7-80a3-3be5f1adb047/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:57.11899739Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2c4cdae1-0bef-46b7-80a3-3be5f1adb047/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:58.064517981Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2c4cdae1-0bef-46b7-80a3-3be5f1adb047/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:50:59.056293854Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2c4cdae1-0bef-46b7-80a3-3be5f1adb047/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:00.434118641Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2c4cdae1-0bef-46b7-80a3-3be5f1adb047/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:01.231079475Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fcf3569-af9e-4967-b82f-54e3a2e91f58): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-06-05T19:51:02.840643192Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:51:03.234071058Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)\"","delay":"23.612361635s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:03.253004267Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41483"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:03.25395758Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"04ee5eff-5816-4d61-a8b6-f0d11ba1e3c1","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41483","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":658,"response_code":"200"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/2c4cdae1-0bef-46b7-80a3-3be5f1adb047/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:03.255876615Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8fcf3569-af9e-4967-b82f-54e3a2e91f58): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:51:03.500258344Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fcf3569-af9e-4967-b82f-54e3a2e91f58): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-06-05T19:51:03.851070386Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8fcf3569-af9e-4967-b82f-54e3a2e91f58): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:51:04.247834222Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"32020f14-eac1-4f14-9ba5-6e7018aabf3e\", \"content-length\": \"173\", \"date\": \"Wed, 05 Jun 2024 19:51:06 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"8fcf3569-af9e-4967-b82f-54e3a2e91f58\\\"\", request_id: \"32020f14-eac1-4f14-9ba5-6e7018aabf3e\" }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:06.200750796Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"20cd6543-01cf-45e3-992a-c188d01a32c8\", \"content-length\": \"173\", \"date\": \"Wed, 05 Jun 2024 19:51:07 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"8fcf3569-af9e-4967-b82f-54e3a2e91f58\\\"\", request_id: \"20cd6543-01cf-45e3-992a-c188d01a32c8\" }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:07.979107928Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:08.264559067Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:08.388936472Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:39388"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:08.390670606Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:08.481017006Z","hostname":"buskin","pid":1555,"uri":"/inventory","method":"GET","req_id":"daae3c92-73f5-40f6-948a-13bdd5347c71","remote_addr":"[fd00:1122:3344:101::a]:39388","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":91984,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:08.48160097Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:08.481667231Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:08.485198675Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:08.48627434Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"4d340c07-1e99-4926-ab03-ab3c2db26115","remote_addr":"[fd00:1122:3344:101::a]:39388","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":4962,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.443978581Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:43686"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.449252166Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 68e120cd-095b-4827-9d1b-7b977952fe73, kind: Service { id: 55f61404-8e20-4047-a779-ffe87d352f7e }, name: Name(\"external-dns-55f61404-8e20-4047-a779-ffe87d352f7e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 7])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b605874a-c92a-4924-97a2-40d87990c249, kind: Service { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175 }, name: Name(\"external-dns-4ab1258a-9fc9-4e50-a387-e0cecec3f175\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 168])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6ec0c20b-4307-47e1-97b9-77d1ea3d29b3, kind: Service { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc }, name: Name(\"nexus-35af6b86-fd74-4104-b900-7a3edf8d03dc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 165, 129])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8e49e841-d58f-44ba-beb8-68b0b0f8fc4c, kind: Service { id: 50145656-8d32-471a-8a33-3c47dc24d267 }, name: Name(\"nexus-50145656-8d32-471a-8a33-3c47dc24d267\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d6d27046-4d61-4670-9a2f-1ffbb00db647, kind: Service { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599 }, name: Name(\"nexus-73e067cf-cfe8-480d-9d5d-fff7aeec0599\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 89])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.449731027Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.449893231Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.449901071Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.449909511Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.449916591Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.449929982Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.450126616Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"632b33a3-d064-42e4-9107-9426131ce9d5","remote_addr":"[fd00:1122:3344:101::a]:43686","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":5973,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.452463871Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:890"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.572063895Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:56582"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.572173048Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.663613396Z","hostname":"buskin","pid":1555,"uri":"/inventory","method":"GET","req_id":"6395104a-8c84-48f6-b1c6-5550f1fe6715","remote_addr":"[fd00:1122:3344:101::a]:56582","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":91449,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.664594219Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.664652151Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.665193773Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.665672535Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"aba9c954-bdd1-4379-aae9-738fce14d1d1","remote_addr":"[fd00:1122:3344:101::a]:56582","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1759,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.668973413Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:65100"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.6692691Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 68e120cd-095b-4827-9d1b-7b977952fe73, kind: Service { id: 55f61404-8e20-4047-a779-ffe87d352f7e }, name: Name(\"external-dns-55f61404-8e20-4047-a779-ffe87d352f7e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 7])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b605874a-c92a-4924-97a2-40d87990c249, kind: Service { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175 }, name: Name(\"external-dns-4ab1258a-9fc9-4e50-a387-e0cecec3f175\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 168])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6ec0c20b-4307-47e1-97b9-77d1ea3d29b3, kind: Service { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc }, name: Name(\"nexus-35af6b86-fd74-4104-b900-7a3edf8d03dc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 165, 129])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8e49e841-d58f-44ba-beb8-68b0b0f8fc4c, kind: Service { id: 50145656-8d32-471a-8a33-3c47dc24d267 }, name: Name(\"nexus-50145656-8d32-471a-8a33-3c47dc24d267\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d6d27046-4d61-4670-9a2f-1ffbb00db647, kind: Service { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599 }, name: Name(\"nexus-73e067cf-cfe8-480d-9d5d-fff7aeec0599\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 89])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.669314111Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.669991867Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.670003847Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.670024368Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.670043518Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.670065969Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.670092449Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"96191054-3c99-4f28-a364-249a62f8e024","remote_addr":"[fd00:1122:3344:101::b]:65100","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":991,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.71374909Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:44303"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.713972055Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 68e120cd-095b-4827-9d1b-7b977952fe73, kind: Service { id: 55f61404-8e20-4047-a779-ffe87d352f7e }, name: Name(\"external-dns-55f61404-8e20-4047-a779-ffe87d352f7e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 7])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b605874a-c92a-4924-97a2-40d87990c249, kind: Service { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175 }, name: Name(\"external-dns-4ab1258a-9fc9-4e50-a387-e0cecec3f175\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 168])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6ec0c20b-4307-47e1-97b9-77d1ea3d29b3, kind: Service { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc }, name: Name(\"nexus-35af6b86-fd74-4104-b900-7a3edf8d03dc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 165, 129])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8e49e841-d58f-44ba-beb8-68b0b0f8fc4c, kind: Service { id: 50145656-8d32-471a-8a33-3c47dc24d267 }, name: Name(\"nexus-50145656-8d32-471a-8a33-3c47dc24d267\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d6d27046-4d61-4670-9a2f-1ffbb00db647, kind: Service { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599 }, name: Name(\"nexus-73e067cf-cfe8-480d-9d5d-fff7aeec0599\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 89])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.714009026Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.714044747Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.714053737Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.714074078Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.714090408Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.714103038Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.714117399Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"53ccda80-9021-4a1b-9671-e2ab0e4b91fb","remote_addr":"[fd00:1122:3344:101::c]:44303","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":260,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.718206505Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:48462"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.718373179Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 68e120cd-095b-4827-9d1b-7b977952fe73, kind: Service { id: 55f61404-8e20-4047-a779-ffe87d352f7e }, name: Name(\"external-dns-55f61404-8e20-4047-a779-ffe87d352f7e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 7])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b605874a-c92a-4924-97a2-40d87990c249, kind: Service { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175 }, name: Name(\"external-dns-4ab1258a-9fc9-4e50-a387-e0cecec3f175\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 168])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6ec0c20b-4307-47e1-97b9-77d1ea3d29b3, kind: Service { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc }, name: Name(\"nexus-35af6b86-fd74-4104-b900-7a3edf8d03dc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 165, 129])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8e49e841-d58f-44ba-beb8-68b0b0f8fc4c, kind: Service { id: 50145656-8d32-471a-8a33-3c47dc24d267 }, name: Name(\"nexus-50145656-8d32-471a-8a33-3c47dc24d267\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d6d27046-4d61-4670-9a2f-1ffbb00db647, kind: Service { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599 }, name: Name(\"nexus-73e067cf-cfe8-480d-9d5d-fff7aeec0599\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 89])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.71839786Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.71841266Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.71843048Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.718436041Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.718442481Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.718450891Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:12.718460451Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"e9b0717a-00be-4f80-b0d6-178cd37d4c73","remote_addr":"[fd00:1122:3344:101::a]:48462","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":169,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:33.286550154Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62531"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:33.287014965Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"8d4f9cc4-1d6a-413e-b5d7-612a1312fa4b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62531","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":464,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:34.448064814Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:45810"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:34.448648028Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"74bbbccd-7ee8-49e4-8abe-8041a858915e","remote_addr":"[fd00:1122:3344:101::b]:45810","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":562,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:34.55083206Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:56403"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:34.552020408Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"f7b44e1e-390a-4440-971f-5f89d2c8a7db","remote_addr":"[fd00:1122:3344:101::a]:56403","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1173,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:34.568698602Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:64251"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:34.568855886Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"58d92fc2-1fbf-464c-ab8c-0039d5535aa6","remote_addr":"[fd00:1122:3344:101::c]:64251","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:34.661751759Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:60478"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:34.770434905Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"c6e8e8a9-8049-4589-b7b8-fd1422653ebc","remote_addr":"[fd00:1122:3344:101::a]:60478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":108637,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:34.771469749Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"ccfc5192-4ff6-4321-9136-42b68fe73c3b","remote_addr":"[fd00:1122:3344:101::a]:60478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":906,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:34.787813885Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:50899"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:34.848656481Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"982bfbab-189c-4241-ae4e-bb9feea964a4","remote_addr":"[fd00:1122:3344:101::b]:50899","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":60806,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:34.848799675Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:47250"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:34.852638406Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"2b2e79f0-d5f0-4681-a51c-2dfb3d0c924e","remote_addr":"[fd00:1122:3344:101::b]:50899","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3611,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:34.853213869Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:34.853808533Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:34.854763696Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:34.8557736Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:34.855818741Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"PUT","req_id":"1b347220-bcfd-4e57-bb64-661fdb121b49","remote_addr":"[fd00:1122:3344:101::a]:60478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":6863,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:34.903530917Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"8f275276-04bc-4aa5-8e5b-58cb824a2ee9","remote_addr":"[fd00:1122:3344:101::c]:47250","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":54644,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:34.903828204Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"bde91a77-f516-421b-95e3-14fd9f74baf2","remote_addr":"[fd00:1122:3344:101::c]:47250","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":126,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:56.97080007Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:59238","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::d]:54832"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:51:56.976421882Z","hostname":"buskin","pid":1555,"uri":"/8fcf3569-af9e-4967-b82f-54e3a2e91f58","method":"GET","req_id":"4ba6cdf2-6ac4-42fe-bd8b-db747e43c367","remote_addr":"[fd00:1122:3344:101::d]:54832","local_addr":"[fd00:1122:3344:101::1]:59238","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":4894,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:03.319055942Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47630"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:03.319152444Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"be5b4e63-735c-4cde-a993-7d2730016c41","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47630","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":91,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:04.522402591Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:62249"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:04.522652617Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"7a2028be-ed00-4e99-aafc-535cbe1527b3","remote_addr":"[fd00:1122:3344:101::b]:62249","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":221,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:04.533069953Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:63450"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:04.533229866Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"b487eb81-c82e-4af8-89b5-e1bb60670462","remote_addr":"[fd00:1122:3344:101::a]:63450","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":125,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:04.617845974Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:35883"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:04.617996938Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"394344ca-cbf2-4037-9706-cd7690ae7b8e","remote_addr":"[fd00:1122:3344:101::c]:35883","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:04.773436768Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:33182"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:04.821714637Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"58154e78-ceea-4509-b605-431a5b1d9d3b","remote_addr":"[fd00:1122:3344:101::a]:33182","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48228,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:04.821995594Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"7d36bfe3-3ed1-4e18-89e3-5d3d65fbeea3","remote_addr":"[fd00:1122:3344:101::a]:33182","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":94,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:04.823216103Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:41071"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:04.875385304Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"1cd718f7-3e26-48c3-a24b-a7130f022377","remote_addr":"[fd00:1122:3344:101::b]:41071","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":52056,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:04.875718852Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"6a2af8c8-c24d-4c8a-9a49-70aa5d206bff","remote_addr":"[fd00:1122:3344:101::b]:41071","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":122,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:04.936418685Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:38021"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:04.980967057Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"eee42459-734e-4cfa-97b0-6d8de8edff44","remote_addr":"[fd00:1122:3344:101::c]:38021","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44463,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:04.981894379Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"1fa6c1ea-a8dc-42ee-bc5f-7d1884e0c651","remote_addr":"[fd00:1122:3344:101::c]:38021","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":782,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:26.971011757Z","hostname":"buskin","pid":1555,"uri":"/8fcf3569-af9e-4967-b82f-54e3a2e91f58","method":"GET","req_id":"59513cf1-5a64-4954-810e-b12dde7d016a","remote_addr":"[fd00:1122:3344:101::d]:54832","local_addr":"[fd00:1122:3344:101::1]:59238","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1593,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:33.353077347Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60921"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:33.353174669Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"0b97ec9f-1162-4aef-809e-4e33ecd5162a","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60921","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":70,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:34.435381498Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:53794"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:34.435601864Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"90c49da0-a015-47af-931a-78a35e46f4c7","remote_addr":"[fd00:1122:3344:101::c]:53794","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":193,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:34.440311075Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:59389"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:34.441169075Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"9e82227f-9e5f-4f00-a5d1-ce9802505c03","remote_addr":"[fd00:1122:3344:101::a]:59389","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":818,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:34.487527929Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:44828"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:34.487659713Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"0cf21d02-26e5-4c73-8e7d-0094956b0695","remote_addr":"[fd00:1122:3344:101::b]:44828","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":91,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:34.538779689Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:58308"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:34.584805016Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"bf74b53a-ff54-468b-a2ad-4c0b2c1dd69a","remote_addr":"[fd00:1122:3344:101::c]:58308","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45981,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:34.585023211Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"85d69390-7fbc-4a63-98db-d4afeea80758","remote_addr":"[fd00:1122:3344:101::c]:58308","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":83,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:34.593186804Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:64484"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:34.635100003Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"61ae17f9-8ca5-4dd6-9e8d-b9705fef2a32","remote_addr":"[fd00:1122:3344:101::a]:64484","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":41811,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:34.635553704Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"acfcfbe3-c7d2-4215-a532-91dc7b4e70ed","remote_addr":"[fd00:1122:3344:101::a]:64484","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":191,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:34.745324376Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:57499"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:34.789098599Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"42cbfeeb-e61a-40ee-ac99-c600843156b9","remote_addr":"[fd00:1122:3344:101::b]:57499","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":43726,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:34.789364675Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"d066896f-2c5d-443a-9689-a2a74d64ac58","remote_addr":"[fd00:1122:3344:101::b]:57499","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":101,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:52:56.969070245Z","hostname":"buskin","pid":1555,"uri":"/8fcf3569-af9e-4967-b82f-54e3a2e91f58","method":"GET","req_id":"c02fc09f-baf1-47e9-8d52-5a93b5bacb39","remote_addr":"[fd00:1122:3344:101::d]:54832","local_addr":"[fd00:1122:3344:101::1]:59238","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":606,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:03.388414343Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54673"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:03.388499885Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"aa71c27b-70d2-4bd5-be87-1a405da269db","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54673","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":84,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:04.617738255Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:42689"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:04.61794566Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"1088c219-72cf-4ccd-8edd-7fa40e8f50ff","remote_addr":"[fd00:1122:3344:101::b]:42689","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":193,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:04.623293006Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:44361"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:04.62342224Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"b692bfda-27f9-4feb-8a69-74208188a9d8","remote_addr":"[fd00:1122:3344:101::c]:44361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:04.66282602Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:39486"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:04.662945242Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"81e62bc7-33e2-43ca-9e71-ad7ee44b87b9","remote_addr":"[fd00:1122:3344:101::a]:39486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":84,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:04.772082629Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:57803"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:04.820942863Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"726efb9f-2210-4e31-83da-9cba0c377f72","remote_addr":"[fd00:1122:3344:101::a]:57803","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48808,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:04.821083016Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:45181"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:04.824448215Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"525a684e-7adb-41ce-81a2-b1302d0c6c62","remote_addr":"[fd00:1122:3344:101::a]:57803","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3159,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:04.829251029Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:40763"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:04.866318134Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"9380fe99-053e-4edd-b7ca-98689ebcbd4d","remote_addr":"[fd00:1122:3344:101::c]:45181","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45120,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:04.870550384Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"70f81e14-626a-41f6-a2fe-db0dbc870165","remote_addr":"[fd00:1122:3344:101::c]:45181","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":588,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:04.912677058Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a3bb1cb8-9b86-4cd5-9216-bb118d0b2774","remote_addr":"[fd00:1122:3344:101::b]:40763","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":82793,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:04.913000046Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"3b14b85e-adb6-49cf-b8c1-3ac59da8117e","remote_addr":"[fd00:1122:3344:101::b]:40763","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":135,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:26.970221655Z","hostname":"buskin","pid":1555,"uri":"/8fcf3569-af9e-4967-b82f-54e3a2e91f58","method":"GET","req_id":"d100ff67-6d11-4c41-be23-fb6f896e7c66","remote_addr":"[fd00:1122:3344:101::d]:54832","local_addr":"[fd00:1122:3344:101::1]:59238","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1135,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:33.419588953Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47591"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:33.419686946Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"b2399d33-42a3-4619-86c3-c2ca68773aaa","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47591","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":81,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:34.436809728Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:64751"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:34.437002793Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"68b930ff-39f4-4544-80ec-d47c390e4b86","remote_addr":"[fd00:1122:3344:101::b]:64751","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":169,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:34.494960781Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:32980"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:34.495120555Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"6712300e-f64c-4998-96ff-1fcab07fde50","remote_addr":"[fd00:1122:3344:101::c]:32980","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:34.586793169Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:49155"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:34.633462621Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"fb11fd68-3ef3-4e27-ac2c-2c9c0949f25c","remote_addr":"[fd00:1122:3344:101::c]:49155","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46620,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:34.633688006Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:42212"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:34.633767088Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:45678"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:34.633942352Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"f6468fa6-5bb4-4cf2-ab0a-2a704b44f8e0","remote_addr":"[fd00:1122:3344:101::a]:42212","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":211,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:34.633957213Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"36d2952f-854f-4567-99da-9435e995f7a4","remote_addr":"[fd00:1122:3344:101::c]:49155","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":274,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:34.681317671Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"5c8608e4-bffd-45af-a4a2-aff9e798b720","remote_addr":"[fd00:1122:3344:101::b]:45678","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47253,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:34.68215788Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"ad737920-110c-40fe-bd0e-4243cad5fc81","remote_addr":"[fd00:1122:3344:101::b]:45678","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":632,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:34.699964481Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:39418"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:34.74440691Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"024bdc90-c240-46a2-b485-195a049494ca","remote_addr":"[fd00:1122:3344:101::a]:39418","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44389,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:34.745039305Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"8082262a-2234-4649-864f-ccfcffdb74ac","remote_addr":"[fd00:1122:3344:101::a]:39418","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":105,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:35.891262316Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:53:56.982372269Z","hostname":"buskin","pid":1555,"uri":"/8fcf3569-af9e-4967-b82f-54e3a2e91f58","method":"GET","req_id":"481a1d8c-498e-467d-a009-2962bf38441f","remote_addr":"[fd00:1122:3344:101::d]:54832","local_addr":"[fd00:1122:3344:101::1]:59238","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":330,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:03.450470324Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39355"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:03.450600047Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"d82dc7ed-896a-4b82-95bb-413fa046b956","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39355","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:04.524592692Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:49383"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:04.525062093Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"825af855-4157-4ac9-8420-ab77c5c20341","remote_addr":"[fd00:1122:3344:101::b]:49383","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":446,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:04.593224622Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:49385"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:04.593404316Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"5de41804-bdd2-4b37-a043-f614f7d3f6d9","remote_addr":"[fd00:1122:3344:101::c]:49385","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":122,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:04.653192088Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:41240"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:04.653337811Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"5c428274-f66a-4e11-ae73-38c030a68670","remote_addr":"[fd00:1122:3344:101::a]:41240","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":89,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:04.778428674Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:42051"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:04.827650196Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"cb7624e2-a422-4d67-a6a3-dba2794c2b58","remote_addr":"[fd00:1122:3344:101::b]:42051","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49150,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:04.828202719Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"63f4bbd8-37eb-47b5-973c-554ea3da5005","remote_addr":"[fd00:1122:3344:101::b]:42051","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:04.84303454Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:54896"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:04.889770143Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"2136949b-a5ac-4a93-8056-45e82c11a94f","remote_addr":"[fd00:1122:3344:101::c]:54896","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46641,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:04.890212163Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"936745b0-0711-49ef-8b5b-6af6d339c6ee","remote_addr":"[fd00:1122:3344:101::c]:54896","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":142,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:04.909963369Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:48271"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:04.956694953Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"f0a40462-a27c-4e19-b90f-8976d72ed59c","remote_addr":"[fd00:1122:3344:101::a]:48271","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46650,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:04.957522802Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"ef515c29-3f23-43a2-888b-c46665c706de","remote_addr":"[fd00:1122:3344:101::a]:48271","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":667,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:27.023019503Z","hostname":"buskin","pid":1555,"uri":"/8fcf3569-af9e-4967-b82f-54e3a2e91f58","method":"GET","req_id":"e8cde10b-9998-4c64-b7d0-ab8e465892dd","remote_addr":"[fd00:1122:3344:101::d]:54832","local_addr":"[fd00:1122:3344:101::1]:59238","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":485,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:33.484951857Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54390"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:33.486081674Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"9dad2cb0-ede0-4198-b9cf-939ae11d8b62","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54390","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1087,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:34.401892294Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:57273"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:34.402747185Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"df6e5037-81f1-4080-9546-0c8ea74256f1","remote_addr":"[fd00:1122:3344:101::b]:57273","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":832,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:34.449112529Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:63382"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:34.449264053Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"1bacc51c-eda3-4ade-b07f-99c783261ee7","remote_addr":"[fd00:1122:3344:101::a]:63382","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":120,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:34.452307355Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:51226"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:34.453156755Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"ea20713c-f59d-4aaa-bf69-7a3323a7d35b","remote_addr":"[fd00:1122:3344:101::c]:51226","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":807,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:34.534214789Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:59658"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:34.581425523Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"0b891956-4475-4da9-9a21-d4e8968d22df","remote_addr":"[fd00:1122:3344:101::b]:59658","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47157,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:34.581677599Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"9297198d-e746-4881-a474-0ce91f1c7c41","remote_addr":"[fd00:1122:3344:101::b]:59658","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":92,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:34.583885251Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:43573"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:34.633573234Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"40a74289-e5d6-4aa4-98b4-20cc906986ac","remote_addr":"[fd00:1122:3344:101::c]:43573","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49589,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:34.633844201Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b958c243-a694-429b-9f80-2270813fe9d8","remote_addr":"[fd00:1122:3344:101::c]:43573","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":88,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:34.705918362Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:61175"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:34.75285159Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"dd04aa9b-c01c-4459-8717-7ad9c8a91b36","remote_addr":"[fd00:1122:3344:101::a]:61175","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46852,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:34.753086026Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b41ac32e-962b-470f-b99b-6fb499467739","remote_addr":"[fd00:1122:3344:101::a]:61175","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":96,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:54:57.024364016Z","hostname":"buskin","pid":1555,"uri":"/8fcf3569-af9e-4967-b82f-54e3a2e91f58","method":"GET","req_id":"486a4878-3585-43fd-9034-cfa43a2247e6","remote_addr":"[fd00:1122:3344:101::d]:54832","local_addr":"[fd00:1122:3344:101::1]:59238","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":2704,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:03.516945312Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50782"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:03.517088425Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"dfe421f3-b5a4-4b49-b0b4-3ac15163b450","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50782","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:04.518935196Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:33908"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:04.519139221Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"2b295fd8-d339-4890-8ee3-9bd961d065d2","remote_addr":"[fd00:1122:3344:101::b]:33908","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":191,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:04.566711844Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:32814"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:04.566862118Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"3a916f46-06c3-47cd-b27d-4d31230fb9b1","remote_addr":"[fd00:1122:3344:101::a]:32814","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":121,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:04.570241898Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:43606"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:04.570427542Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"78cd0f52-cf79-475f-87af-b94e69be1364","remote_addr":"[fd00:1122:3344:101::c]:43606","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:04.770828293Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:48795"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:04.818944219Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a1d6ed17-e10e-4f61-8735-5467d2ed9adf","remote_addr":"[fd00:1122:3344:101::b]:48795","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48054,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:04.819628425Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"489cf5c2-7e83-4cb6-be18-8eda99c5a928","remote_addr":"[fd00:1122:3344:101::b]:48795","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":542,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:04.822179415Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:43130"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:04.864494684Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"7c63c844-45ca-4c7b-a2a9-718ef6770840","remote_addr":"[fd00:1122:3344:101::a]:43130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":42220,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:04.8647601Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"870c430a-2f26-4d15-879c-2d64278c5583","remote_addr":"[fd00:1122:3344:101::a]:43130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":96,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:04.968468792Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:42392"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:05.010306627Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"fe67dfd3-7b5f-4130-822e-a66b44b7e61a","remote_addr":"[fd00:1122:3344:101::c]:42392","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":41710,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:05.010602483Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b5a63fe0-9c5f-4de8-b32a-3a5afd81e59d","remote_addr":"[fd00:1122:3344:101::c]:42392","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":96,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:27.023375947Z","hostname":"buskin","pid":1555,"uri":"/8fcf3569-af9e-4967-b82f-54e3a2e91f58","method":"GET","req_id":"e0a6dc9c-fdb0-41ec-a54a-06d7bc6f27b0","remote_addr":"[fd00:1122:3344:101::d]:54832","local_addr":"[fd00:1122:3344:101::1]:59238","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":542,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:33.54846505Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65459"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:33.548565902Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"4990f1e4-d2d0-48c7-893e-e95410e245cb","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65459","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":75,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:34.421987701Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:48594"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:34.422185606Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"50288319-bfa8-4f36-a8f7-2822b3eb45a6","remote_addr":"[fd00:1122:3344:101::b]:48594","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":174,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:34.527042772Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:45306"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:34.527192215Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"2eacd02e-2979-4318-a1ed-d9ad1a529fac","remote_addr":"[fd00:1122:3344:101::a]:45306","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":128,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:34.6125245Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:44437"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:34.613512553Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"1b57bba5-8bbc-4103-bc26-afd72cfe9e5c","remote_addr":"[fd00:1122:3344:101::c]:44437","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":967,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:34.630432173Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:45091"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:34.676417418Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"f61a1284-ea2b-45b7-a699-2345290a2964","remote_addr":"[fd00:1122:3344:101::a]:45091","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45938,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:34.676646744Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"036e58c3-65f3-42c0-b48e-a3371192bf98","remote_addr":"[fd00:1122:3344:101::a]:45091","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":78,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:34.699722128Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:59814"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:34.74552475Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"ae9e7047-933a-4278-a784-cabf17f46beb","remote_addr":"[fd00:1122:3344:101::c]:59814","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45702,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:34.745677483Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:33060"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:34.748895619Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"bf312064-de01-42a9-b37e-c020967f7219","remote_addr":"[fd00:1122:3344:101::c]:59814","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3026,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:34.789953048Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"43f58ad3-81f7-44c7-bb73-1d5ea4d5aedc","remote_addr":"[fd00:1122:3344:101::b]:33060","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44169,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:34.790316227Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"9fb353e2-47af-4dfa-b64b-f827dc37b9bd","remote_addr":"[fd00:1122:3344:101::b]:33060","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":159,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:55:57.022718685Z","hostname":"buskin","pid":1555,"uri":"/8fcf3569-af9e-4967-b82f-54e3a2e91f58","method":"GET","req_id":"ba161b89-55bd-4f28-bd70-2d6afa3558a3","remote_addr":"[fd00:1122:3344:101::d]:54832","local_addr":"[fd00:1122:3344:101::1]:59238","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":485,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:03.580989316Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46075"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:03.581120919Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"228ad2e7-0f1d-4b8d-806a-23ae9278db75","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46075","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":129,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.473054405Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:53339"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.47328412Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"1a23dfb6-2a48-4bf6-8f29-372f1d29920b","remote_addr":"[fd00:1122:3344:101::b]:53339","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":214,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.682060549Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:64682"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.682268134Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 68e120cd-095b-4827-9d1b-7b977952fe73, kind: Service { id: 55f61404-8e20-4047-a779-ffe87d352f7e }, name: Name(\"external-dns-55f61404-8e20-4047-a779-ffe87d352f7e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 7])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b605874a-c92a-4924-97a2-40d87990c249, kind: Service { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175 }, name: Name(\"external-dns-4ab1258a-9fc9-4e50-a387-e0cecec3f175\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 168])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6ec0c20b-4307-47e1-97b9-77d1ea3d29b3, kind: Service { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc }, name: Name(\"nexus-35af6b86-fd74-4104-b900-7a3edf8d03dc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 165, 129])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8e49e841-d58f-44ba-beb8-68b0b0f8fc4c, kind: Service { id: 50145656-8d32-471a-8a33-3c47dc24d267 }, name: Name(\"nexus-50145656-8d32-471a-8a33-3c47dc24d267\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d6d27046-4d61-4670-9a2f-1ffbb00db647, kind: Service { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599 }, name: Name(\"nexus-73e067cf-cfe8-480d-9d5d-fff7aeec0599\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 89])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.682339175Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.682378576Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.682386656Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.682418217Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.682435168Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.682447298Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.682475329Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"7f10be0d-8ed8-4a33-9043-71a6c3d1a042","remote_addr":"[fd00:1122:3344:101::b]:64682","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":372,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.70453941Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:48616"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.705540443Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"ec5801d0-3398-4314-a39a-d0562531ad2b","remote_addr":"[fd00:1122:3344:101::a]:48616","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":982,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.726700823Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:59416"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.726825445Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 68e120cd-095b-4827-9d1b-7b977952fe73, kind: Service { id: 55f61404-8e20-4047-a779-ffe87d352f7e }, name: Name(\"external-dns-55f61404-8e20-4047-a779-ffe87d352f7e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 7])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b605874a-c92a-4924-97a2-40d87990c249, kind: Service { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175 }, name: Name(\"external-dns-4ab1258a-9fc9-4e50-a387-e0cecec3f175\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 168])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6ec0c20b-4307-47e1-97b9-77d1ea3d29b3, kind: Service { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc }, name: Name(\"nexus-35af6b86-fd74-4104-b900-7a3edf8d03dc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 165, 129])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8e49e841-d58f-44ba-beb8-68b0b0f8fc4c, kind: Service { id: 50145656-8d32-471a-8a33-3c47dc24d267 }, name: Name(\"nexus-50145656-8d32-471a-8a33-3c47dc24d267\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d6d27046-4d61-4670-9a2f-1ffbb00db647, kind: Service { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599 }, name: Name(\"nexus-73e067cf-cfe8-480d-9d5d-fff7aeec0599\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 89])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.726863006Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.726886007Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.726892697Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.726907398Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.726923688Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.726946078Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.726959689Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"fcc24846-9ee9-4a54-acc4-7a7adef12000","remote_addr":"[fd00:1122:3344:101::a]:59416","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":227,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.779139891Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:34478"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.779345336Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"8251be4a-d530-4f92-918a-0783335dc2ce","remote_addr":"[fd00:1122:3344:101::c]:34478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":116,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.853797753Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:33586"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.903509887Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"1eaa4af4-d3a1-4ff7-b567-f1fd888618af","remote_addr":"[fd00:1122:3344:101::a]:33586","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49643,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.90366783Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:59601"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.903766393Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:60872"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.904011099Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 68e120cd-095b-4827-9d1b-7b977952fe73, kind: Service { id: 55f61404-8e20-4047-a779-ffe87d352f7e }, name: Name(\"external-dns-55f61404-8e20-4047-a779-ffe87d352f7e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 192, 7])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b605874a-c92a-4924-97a2-40d87990c249, kind: Service { id: 4ab1258a-9fc9-4e50-a387-e0cecec3f175 }, name: Name(\"external-dns-4ab1258a-9fc9-4e50-a387-e0cecec3f175\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 168])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6ec0c20b-4307-47e1-97b9-77d1ea3d29b3, kind: Service { id: 35af6b86-fd74-4104-b900-7a3edf8d03dc }, name: Name(\"nexus-35af6b86-fd74-4104-b900-7a3edf8d03dc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 165, 129])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8e49e841-d58f-44ba-beb8-68b0b0f8fc4c, kind: Service { id: 50145656-8d32-471a-8a33-3c47dc24d267 }, name: Name(\"nexus-50145656-8d32-471a-8a33-3c47dc24d267\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d6d27046-4d61-4670-9a2f-1ffbb00db647, kind: Service { id: 73e067cf-cfe8-480d-9d5d-fff7aeec0599 }, name: Name(\"nexus-73e067cf-cfe8-480d-9d5d-fff7aeec0599\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 89])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.90408938Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.904148912Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"8158b055-b2a7-4686-809a-8da433c49c84","remote_addr":"[fd00:1122:3344:101::a]:33586","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":290,"response_code":"200"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.904174482Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.904186973Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.904222113Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.904262264Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.904286735Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.904303695Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"982318af-8748-404c-89f9-40bbcd1adf5a","remote_addr":"[fd00:1122:3344:101::c]:59601","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":561,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.951407407Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"5383d0a7-b680-4de4-a765-b3e8f3e77209","remote_addr":"[fd00:1122:3344:101::b]:60872","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47488,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:04.951706985Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"7c7158fb-e161-461d-9400-77c9e8930398","remote_addr":"[fd00:1122:3344:101::b]:60872","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":114,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:05.038135275Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:51928"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:05.082988824Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b0419926-64d3-4d3c-9ae0-7f0c81415bfd","remote_addr":"[fd00:1122:3344:101::c]:51928","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44773,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:05.083709701Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"799e2b05-dbaa-43dc-af44-eee960ff7ec2","remote_addr":"[fd00:1122:3344:101::c]:51928","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":577,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:27.024052059Z","hostname":"buskin","pid":1555,"uri":"/8fcf3569-af9e-4967-b82f-54e3a2e91f58","method":"GET","req_id":"74300c14-9a57-4654-ab93-e8f4376c8a9d","remote_addr":"[fd00:1122:3344:101::d]:54832","local_addr":"[fd00:1122:3344:101::1]:59238","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":505,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:33.615485299Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49259"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:33.615634473Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"c6aad4fb-bcbd-49dc-bb91-6672ba52c53a","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49259","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:34.431714848Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:54207"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:34.431917903Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"1a9366c7-dc90-43ff-bb0f-adf27c9c9dc0","remote_addr":"[fd00:1122:3344:101::a]:54207","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":180,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:34.537601828Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:39171"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:34.537793053Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"79d34674-89af-4019-957c-bb125deaa5ef","remote_addr":"[fd00:1122:3344:101::b]:39171","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":173,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:34.61389938Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:45094"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:34.614029973Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"459f2b3b-bdee-407f-91f3-cc2c27461805","remote_addr":"[fd00:1122:3344:101::c]:45094","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:34.640063977Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:36984"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:34.687421536Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"55b1bd63-1618-46d3-891d-c283369897aa","remote_addr":"[fd00:1122:3344:101::a]:36984","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47298,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:34.687642271Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:49544"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:34.687710202Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:35189"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:34.687910437Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"31c39c04-6da4-4c11-b3b4-ea9ef29b78b3","remote_addr":"[fd00:1122:3344:101::a]:36984","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":268,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:34.728620798Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"8d3797a6-6d3d-4dd3-9618-638d0fc7dbc9","remote_addr":"[fd00:1122:3344:101::c]:49544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":40851,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:34.732309465Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"824c967c-646c-4200-878a-3ff0ec525366","remote_addr":"[fd00:1122:3344:101::c]:49544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":546,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:34.771171863Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"5ceed8cb-f13f-40b1-a3ab-cb7b1e8ad642","remote_addr":"[fd00:1122:3344:101::b]:35189","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":83345,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:34.771571502Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"817bea6f-1f06-4965-9b63-018a8eb23160","remote_addr":"[fd00:1122:3344:101::b]:35189","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":158,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:46.240170273Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:53936"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:46.240362597Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"79667103-adbc-451d-86d9-d20be76c4da9","remote_addr":"[fd00:1122:3344:101::a]:53936","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":179,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:46.243286406Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:631","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 244, 1, 68])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(28700) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:46.243325427Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"PUT","req_id":"0cba5c1d-e431-4171-8df8-8610b4956e17","remote_addr":"[fd00:1122:3344:101::a]:53936","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":2803,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:46.408409674Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:57817"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:46.420461849Z","hostname":"buskin","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:546","metadata":"InstanceMetadata { silo_id: 1c3d7737-cab3-45be-85fd-26b8750520fe, project_id: d1af6593-b5f1-4200-81cd-9b9a32cb9da3 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-06-05T19:56:45.953Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(b6eaa280-c5b9-4310-a8af-92e7aede12b7), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-05T19:56:46.003056245Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 50f59e69-eba1-4727-87c4-d8dd227ef3fd, kind: Instance { id: f98ff033-98f9-40d8-971e-082541bbd5e4 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 1, 68])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(28700), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 50f59e69-eba1-4727-87c4-d8dd227ef3fd, kind: Instance { id: f98ff033-98f9-40d8-971e-082541bbd5e4 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 1, 68])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(28700), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 50f59e69-eba1-4727-87c4-d8dd227ef3fd, kind: Instance { id: f98ff033-98f9-40d8-971e-082541bbd5e4 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 1, 68])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(28700), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(28700))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 50f59e69-eba1-4727-87c4-d8dd227ef3fd, kind: Instance { id: f98ff033-98f9-40d8-971e-082541bbd5e4 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 1, 68])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(28700), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-8d976ff1396\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 506a7161-9d79-4e8b-9d88-b463bf99b783, read_only_parent: Some(Volume { block_size: 512, id: 40268bf5-852c-468d-be9c-3a34676786f7, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 5f274592-a2b1-4165-915b-3b05748c10df, key: Some(\"pccUBQlmS7alObm2D+/kuVoQTft9cFTF22BUsE0Z7do=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::19]:19002\", \"[fd00:1122:3344:101::18]:19001\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 506a7161-9d79-4e8b-9d88-b463bf99b783, key: Some(\"ZxUDQpPrLbX1Yga+ihh5/XKgAAGcut1ZHmYmJLiJUKg=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::14]:19002\", \"[fd00:1122:3344:101::17]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAAlF5/FWMVYAAAXn8VYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAJRefxVjFWAAAF5/FWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJmOThmZjAzMy05OGY5LTQwZDgtOTcxZS0wODI1NDFiYmQ1ZTQiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSWlVZVNZcTB4OTU4R3NwU2FxZkI0L1ZxdlcwTHp2dVRzclVuNFBmNlM5biJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"b6eaa280-c5b9-4310-a8af-92e7aede12b7","instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:46.420567681Z","hostname":"buskin","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:580","instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:46.422054106Z","hostname":"buskin","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance.rs:938","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 50f59e69-eba1-4727-87c4-d8dd227ef3fd, kind: Instance { id: f98ff033-98f9-40d8-971e-082541bbd5e4 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 1, 68])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(28700), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 50f59e69-eba1-4727-87c4-d8dd227ef3fd, kind: Instance { id: f98ff033-98f9-40d8-971e-082541bbd5e4 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 1, 68])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(28700), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 50f59e69-eba1-4727-87c4-d8dd227ef3fd, kind: Instance { id: f98ff033-98f9-40d8-971e-082541bbd5e4 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 1, 68])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(28700), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(28700))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 50f59e69-eba1-4727-87c4-d8dd227ef3fd, kind: Instance { id: f98ff033-98f9-40d8-971e-082541bbd5e4 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 1, 68])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(28700), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-8d976ff1396\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 506a7161-9d79-4e8b-9d88-b463bf99b783, read_only_parent: Some(Volume { block_size: 512, id: 40268bf5-852c-468d-be9c-3a34676786f7, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 5f274592-a2b1-4165-915b-3b05748c10df, key: Some(\"pccUBQlmS7alObm2D+/kuVoQTft9cFTF22BUsE0Z7do=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::19]:19002\", \"[fd00:1122:3344:101::18]:19001\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 506a7161-9d79-4e8b-9d88-b463bf99b783, key: Some(\"ZxUDQpPrLbX1Yga+ihh5/XKgAAGcut1ZHmYmJLiJUKg=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::14]:19002\", \"[fd00:1122:3344:101::17]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAAlF5/FWMVYAAAXn8VYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAJRefxVjFWAAAF5/FWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJmOThmZjAzMy05OGY5LTQwZDgtOTcxZS0wODI1NDFiYmQ1ZTQiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSWlVZVNZcTB4OTU4R3NwU2FxZkI0L1ZxdlcwTHp2dVRzclVuNFBmNlM5biJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(b6eaa280-c5b9-4310-a8af-92e7aede12b7), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-05T19:56:46.003056245Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-06-05T19:56:45.953Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"b6eaa280-c5b9-4310-a8af-92e7aede12b7","instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:46.42391937Z","hostname":"buskin","pid":1555,"uri":"/instances/f98ff033-98f9-40d8-971e-082541bbd5e4","method":"PUT","req_id":"36b5ec0f-05b1-4be2-b99a-c4d792217c90","remote_addr":"[fd00:1122:3344:101::a]:57817","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":14264,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:46.509545862Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:64259"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:46.5120123Z","hostname":"buskin","pid":1555,"uri":"/instances/f98ff033-98f9-40d8-971e-082541bbd5e4/state","method":"PUT","req_id":"fa934c28-582f-4e72-a803-fbed25927eea","remote_addr":"[fd00:1122:3344:101::a]:64259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":2405,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:46.535084604Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 244, 1, 68]), slot: 0, vni: Vni { inner: 28700 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:46.536402176Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_b6eaa280-c5b9-4310-a8af-92e7aede12b7","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:46.568647947Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_b6eaa280-c5b9-4310-a8af-92e7aede12b7","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:46.590557764Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_b6eaa280-c5b9-4310-a8af-92e7aede12b7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:48.31946966Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:48.319762037Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_b6eaa280-c5b9-4310-a8af-92e7aede12b7","instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_propolis-server_b6eaa280-c5b9-4310-a8af-92e7aede12b7\") failed: Property not found. retry in 68.358998ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:56:48.524840389Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_b6eaa280-c5b9-4310-a8af-92e7aede12b7","instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_b6eaa280-c5b9-4310-a8af-92e7aede12b7\") failed: Property not found. retry in 129.466901ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:56:48.607287495Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_b6eaa280-c5b9-4310-a8af-92e7aede12b7","instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_b6eaa280-c5b9-4310-a8af-92e7aede12b7\") failed: Property not found. retry in 259.628034ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:56:48.758977776Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_b6eaa280-c5b9-4310-a8af-92e7aede12b7","instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_b6eaa280-c5b9-4310-a8af-92e7aede12b7\") failed: Property not found. retry in 342.174479ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:56:49.034574473Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_b6eaa280-c5b9-4310-a8af-92e7aede12b7","instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_b6eaa280-c5b9-4310-a8af-92e7aede12b7\") failed: Property not found. retry in 893.156451ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:56:49.390128496Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_b6eaa280-c5b9-4310-a8af-92e7aede12b7","instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_b6eaa280-c5b9-4310-a8af-92e7aede12b7\") failed: Property not found. retry in 1.310025089s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:56:50.294124868Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_b6eaa280-c5b9-4310-a8af-92e7aede12b7","instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_b6eaa280-c5b9-4310-a8af-92e7aede12b7\") failed: Property not found. retry in 1.183520598s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:56:51.614406487Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_b6eaa280-c5b9-4310-a8af-92e7aede12b7","instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_b6eaa280-c5b9-4310-a8af-92e7aede12b7\") failed: Property not found. retry in 1.434551144s","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:56:52.807751099Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_b6eaa280-c5b9-4310-a8af-92e7aede12b7","instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_b6eaa280-c5b9-4310-a8af-92e7aede12b7","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:54.269487918Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"sled-agent/src/instance.rs:1423"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:54.277568059Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"sled-agent/src/instance.rs:1432"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:54.311170432Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"sled-agent/src/instance.rs:1446"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAAlF5/FWMVYAAAXn8VYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAJRefxVjFWAAAF5/FWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJmOThmZjAzMy05OGY5LTQwZDgtOTcxZS0wODI1NDFiYmQ1ZTQiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSWlVZVNZcTB4OTU4R3NwU2FxZkI0L1ZxdlcwTHp2dVRzclVuNFBmNlM5biJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-8d976ff1396\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 506a7161-9d79-4e8b-9d88-b463bf99b783, read_only_parent: Some(Volume { block_size: 512, id: 40268bf5-852c-468d-be9c-3a34676786f7, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 5f274592-a2b1-4165-915b-3b05748c10df, key: Some(\"pccUBQlmS7alObm2D+/kuVoQTft9cFTF22BUsE0Z7do=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::19]:19002\", \"[fd00:1122:3344:101::18]:19001\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 506a7161-9d79-4e8b-9d88-b463bf99b783, key: Some(\"ZxUDQpPrLbX1Yga+ihh5/XKgAAGcut1ZHmYmJLiJUKg=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::14]:19002\", \"[fd00:1122:3344:101::17]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: f98ff033-98f9-40d8-971e-082541bbd5e4, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: d1af6593-b5f1-4200-81cd-9b9a32cb9da3, silo_id: 1c3d7737-cab3-45be-85fd-26b8750520fe }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:54.315457253Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"sled-agent/src/instance.rs:685"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:54.404853933Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"sled-agent/src/instance.rs:687"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-06-05T19:56:54.406352249Z }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:54.407198439Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:54.407219249Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-06-05T19:56:54.406352249Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(b6eaa280-c5b9-4310-a8af-92e7aede12b7), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-05T19:56:46.003056245Z }","propolis_id":"b6eaa280-c5b9-4310-a8af-92e7aede12b7"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:54.409027192Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(b6eaa280-c5b9-4310-a8af-92e7aede12b7), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-05T19:56:46.003056245Z }, propolis_id: b6eaa280-c5b9-4310-a8af-92e7aede12b7, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-06-05T19:56:54.406352249Z } }","instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-06-05T19:56:54.445497093Z }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:54.445512524Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:54.445546414Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-06-05T19:56:54.445497093Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(b6eaa280-c5b9-4310-a8af-92e7aede12b7), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-05T19:56:46.003056245Z }","propolis_id":"b6eaa280-c5b9-4310-a8af-92e7aede12b7"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:54.445553274Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(b6eaa280-c5b9-4310-a8af-92e7aede12b7), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-05T19:56:46.003056245Z }, propolis_id: b6eaa280-c5b9-4310-a8af-92e7aede12b7, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-06-05T19:56:54.445497093Z } }","instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:56:57.128999505Z","hostname":"buskin","pid":1555,"uri":"/8fcf3569-af9e-4967-b82f-54e3a2e91f58","method":"GET","req_id":"7ddc7db3-b78f-4636-9c47-ce1e0a6dbdc7","remote_addr":"[fd00:1122:3344:101::d]:54832","local_addr":"[fd00:1122:3344:101::1]:59238","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8fcf3569-af9e-4967-b82f-54e3a2e91f58","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":580,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:03.648859304Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60584"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:03.648992637Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"4892ac16-4202-4005-98a6-a20be23f5641","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60584","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":87,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:03.869591245Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:34258"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:03.870381303Z","hostname":"buskin","pid":1555,"uri":"/instances/f98ff033-98f9-40d8-971e-082541bbd5e4/state","method":"PUT","req_id":"33b4e62c-102c-4b33-a91b-8b89151aba2f","remote_addr":"[fd00:1122:3344:101::a]:34258","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":714,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-06-05T19:57:03.870496006Z }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:03.870523997Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:03.870538687Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-06-05T19:57:03.870496006Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(b6eaa280-c5b9-4310-a8af-92e7aede12b7), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-05T19:56:46.003056245Z }","propolis_id":"b6eaa280-c5b9-4310-a8af-92e7aede12b7"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:03.870556928Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(b6eaa280-c5b9-4310-a8af-92e7aede12b7), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-05T19:56:46.003056245Z }, propolis_id: b6eaa280-c5b9-4310-a8af-92e7aede12b7, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-06-05T19:57:03.870496006Z } }","instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-06-05T19:57:03.911840552Z }","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:03.911861633Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:03.911882873Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-06-05T19:57:03.911840552Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-06-05T19:57:03.911840552Z }","propolis_id":"b6eaa280-c5b9-4310-a8af-92e7aede12b7"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:03.911900133Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:03.911921044Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:03.91387046Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:03.915327874Z","hostname":"buskin","pid":1555,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:437","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/bcb94b78-d8ca-43f0-a9d3-d42c4b7857e7/debug/bundle/zone\", \"/pool/int/ca6bd534-e33c-4c83-bc67-7b70494fb021/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/906c9449-467f-496f-ae4c-f2197a847844/crypt/debug\", \"/pool/ext/d69c4ddc-54d3-4718-a115-7582c8252249/crypt/debug\", \"/pool/ext/8e7826e7-938b-4b78-bd9b-c82573a27350/crypt/debug\", \"/pool/ext/87fce040-2c58-4185-8fd5-7dfb9378086a/crypt/debug\", \"/pool/ext/906c0a61-9cfc-4b51-be6b-6990f707b063/crypt/debug\", \"/pool/ext/ca57da68-282d-4e81-9a3e-bfb822551ad8/crypt/debug\", \"/pool/ext/df264fcd-b75d-4ee3-9b2e-e281836815f9/crypt/debug\", \"/pool/ext/9fef4b9e-a443-49fc-b47e-d8cdc87127ad/crypt/debug\", \"/pool/ext/c495d42b-321e-4420-8635-102c37b30fb2/crypt/debug\"] }","zone_name":"oxz_propolis-server_b6eaa280-c5b9-4310-a8af-92e7aede12b7"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:04.527563227Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:45845"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:04.60773986Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:36706"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:04.618156956Z","hostname":"buskin","pid":1555,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1230","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_b6eaa280-c5b9-4310-a8af-92e7aede12b7\", bundle_id: 3611c987-1f5c-4580-abad-bb3c7ba1006c }, time_created: 2024-06-05T19:57:03.915475978Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_b6eaa280-c5b9-4310-a8af-92e7aede12b7","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:57:04.618248108Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:04.625661073Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:49164"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:04.626014962Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"dfb1cfc7-9fac-4356-a5f6-1d2d562764b1","remote_addr":"[fd00:1122:3344:101::a]:49164","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":163,"response_code":"200"} {"msg":"Clearing mapping of virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:04.627785524Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 244, 1, 68])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(28700) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:04.627834015Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"DELETE","req_id":"6b04a95f-30b0-4f45-8317-2f3905cf8192","remote_addr":"[fd00:1122:3344:101::a]:49164","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1592,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:04.642555952Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:48660"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:04.642756367Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"401d9937-404b-4663-9171-604cd1916cb5","remote_addr":"[fd00:1122:3344:101::b]:48660","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":159,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:04.643544487Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:51955"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:04.643741348Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"b7526be2-c604-4312-928c-015181c5d9a6","remote_addr":"[fd00:1122:3344:101::c]:51955","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:04.784233767Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:64108"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:04.833273245Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:45916"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:04.878585654Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"d2b473c7-0c73-49ec-903b-e3a01927a4b5","remote_addr":"[fd00:1122:3344:101::a]:45916","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45232,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:04.878936273Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"a09f7a23-a872-41ca-9948-f723e92b69cb","remote_addr":"[fd00:1122:3344:101::a]:45916","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":129,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:04.924027447Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:53136"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:04.966221313Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"e94ce55c-c892-4a80-9436-d1dc51e8bc8b","remote_addr":"[fd00:1122:3344:101::b]:53136","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":42131,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:04.9728583Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"e2a936e8-bd86-4390-a7a0-f68621e5b137","remote_addr":"[fd00:1122:3344:101::b]:53136","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":804,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:04.976281991Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:47703"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:05.01815408Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"9e623022-438c-4791-9205-13512edc58c0","remote_addr":"[fd00:1122:3344:101::c]:47703","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":41784,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:05.037153238Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"2737e60e-7403-4904-9bb7-e1613b3ba3c3","remote_addr":"[fd00:1122:3344:101::c]:47703","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":765,"response_code":"200"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:06.365059406Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:06.388894759Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-06-05T19:57:03.911840552Z }, propolis_id: b6eaa280-c5b9-4310-a8af-92e7aede12b7, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-06-05T19:57:06.388828758Z } }","instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:06.481934466Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:42282"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:06.482177721Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"64541fbd-1be6-4752-a09b-825cdf71fab4","remote_addr":"[fd00:1122:3344:101::a]:42282","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":192,"response_code":"200"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:06.785968433Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-06-05T19:57:03.911840552Z }, propolis_id: b6eaa280-c5b9-4310-a8af-92e7aede12b7, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-06-05T19:57:06.388828758Z } }","instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:06.789329743Z","hostname":"buskin","pid":1555,"instance_id":"f98ff033-98f9-40d8-971e-082541bbd5e4","component":"InstanceManager","file":"sled-agent/src/instance.rs:723"} {"msg":"Error handling request","v":0,"name":"SledAgent","level":40,"time":"2024-06-05T19:57:06.812779806Z","hostname":"buskin","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:502","err":"Instance(RequestDropped(RecvError(())))","request":"GetState"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:06.816059374Z","hostname":"buskin","pid":1555,"uri":"/instances/f98ff033-98f9-40d8-971e-082541bbd5e4/state","method":"GET","req_id":"28067b04-3fdd-4bcf-95cd-c8596530cf7c","remote_addr":"[fd00:1122:3344:101::a]:36706","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Not Found","error_message_internal":"No such instance ID: f98ff033-98f9-40d8-971e-082541bbd5e4","latency_us":2205703,"response_code":"404"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:06.816077704Z","hostname":"buskin","pid":1555,"uri":"/instances/f98ff033-98f9-40d8-971e-082541bbd5e4/state","method":"GET","req_id":"f03398fe-1a54-47f8-a46d-558d08ada2a7","remote_addr":"[fd00:1122:3344:101::c]:64108","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Not Found","error_message_internal":"No such instance ID: f98ff033-98f9-40d8-971e-082541bbd5e4","latency_us":2029698,"response_code":"404"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:06.816087494Z","hostname":"buskin","pid":1555,"uri":"/instances/f98ff033-98f9-40d8-971e-082541bbd5e4/state","method":"GET","req_id":"c2210310-2b30-4fda-bc4c-6871537040bf","remote_addr":"[fd00:1122:3344:101::b]:45845","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Error managing instances: Instance Manager dropped our request","latency_us":2286333,"response_code":"500"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:08.146990344Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:56733"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-05T19:57:08.147174349Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"1ae211c2-df6d-48d5-ad96-baae73435b23","remote_addr":"[fd00:1122:3344:101::a]:56733","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":161,"response_code":"200"}