[ May 8 03:32:09 Enabled. ] [ May 8 03:32:09 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ May 8 03:32:09 Method "start" exited with status 0. ] [ May 8 03:32:09 Rereading configuration. ] [ May 8 03:32:09 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:09.661036772Z","hostname":"buskin","pid":1593,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:09.661196093Z","hostname":"buskin","pid":1593,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:09.714274815Z","hostname":"buskin","pid":1593,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:09.714325805Z","hostname":"buskin","pid":1593,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:09.825933891Z","hostname":"buskin","pid":1593,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:09.835076565Z","hostname":"buskin","pid":1593,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:09.842641559Z","hostname":"buskin","pid":1593,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.447289433Z","hostname":"buskin","pid":1593,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:9fb2::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.587283586Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/long_running_tasks.rs:119"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.587377617Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/long_running_tasks.rs:131"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.587390647Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.587483017Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/long_running_tasks.rs:162","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.587494288Z","hostname":"buskin","pid":1593,"component":"DumpSetup-archival","file":"sled-agent/src/dump_setup.rs:291"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.587507528Z","hostname":"buskin","pid":1593,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.587522388Z","hostname":"buskin","pid":1593,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:625"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.590446775Z","hostname":"buskin","pid":1593,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:10.590466085Z","hostname":"buskin","pid":1593,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:674"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.590473595Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.590480565Z","hostname":"buskin","pid":1593,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:733"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.590494765Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.590552836Z","hostname":"buskin","pid":1593,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.590690587Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.590713047Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.590747137Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.602488026Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.602537486Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"80ce766d-f6eb-440d-b6a8-ad5a1011c02b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.698138458Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: 80ce766d-f6eb-440d-b6a8-ad5a1011c02b (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.901160492Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.901198692Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.901204292Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.901214542Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.901220172Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.901237372Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.901241902Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.909882953Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.909910533Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"b3bd066e-6f80-4f80-bc0d-64ec35b22d85 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:10.999204668Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: b3bd066e-6f80-4f80-bc0d-64ec35b22d85 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201257416Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201306336Z","hostname":"buskin","pid":1593,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 80ce766d-f6eb-440d-b6a8-ad5a1011c02b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: b3bd066e-6f80-4f80-bc0d-64ec35b22d85 (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201374777Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201382117Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201423927Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_1.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201457387Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201462747Z","hostname":"buskin","pid":1593,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 80ce766d-f6eb-440d-b6a8-ad5a1011c02b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: b3bd066e-6f80-4f80-bc0d-64ec35b22d85 (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201475977Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201481677Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201485907Z","hostname":"buskin","pid":1593,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 80ce766d-f6eb-440d-b6a8-ad5a1011c02b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: b3bd066e-6f80-4f80-bc0d-64ec35b22d85 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201510727Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201519077Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201526297Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201535618Z","hostname":"buskin","pid":1593,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 80ce766d-f6eb-440d-b6a8-ad5a1011c02b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: b3bd066e-6f80-4f80-bc0d-64ec35b22d85 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201555998Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201566638Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201596288Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_4.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201614338Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201619288Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201625258Z","hostname":"buskin","pid":1593,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 80ce766d-f6eb-440d-b6a8-ad5a1011c02b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: b3bd066e-6f80-4f80-bc0d-64ec35b22d85 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201642938Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_5.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201656858Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201661498Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201666758Z","hostname":"buskin","pid":1593,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 80ce766d-f6eb-440d-b6a8-ad5a1011c02b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: b3bd066e-6f80-4f80-bc0d-64ec35b22d85 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201685168Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201690178Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201694958Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201700199Z","hostname":"buskin","pid":1593,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 80ce766d-f6eb-440d-b6a8-ad5a1011c02b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: b3bd066e-6f80-4f80-bc0d-64ec35b22d85 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201719168Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201724408Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201736589Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201742189Z","hostname":"buskin","pid":1593,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 80ce766d-f6eb-440d-b6a8-ad5a1011c02b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: b3bd066e-6f80-4f80-bc0d-64ec35b22d85 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201765029Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201770319Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201774899Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201780889Z","hostname":"buskin","pid":1593,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 80ce766d-f6eb-440d-b6a8-ad5a1011c02b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: b3bd066e-6f80-4f80-bc0d-64ec35b22d85 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201808209Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/long_running_tasks.rs:91"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201819769Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/long_running_tasks.rs:93"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201824009Z","hostname":"buskin","pid":1593,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 80ce766d-f6eb-440d-b6a8-ad5a1011c02b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: b3bd066e-6f80-4f80-bc0d-64ec35b22d85 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201847989Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.201865969Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"No ledger in /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.20192282Z","hostname":"buskin","pid":1593,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.20193448Z","hostname":"buskin","pid":1593,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.20194393Z","hostname":"buskin","pid":1593,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.2019542Z","hostname":"buskin","pid":1593,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.20196834Z","hostname":"buskin","pid":1593,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.20198775Z","hostname":"buskin","pid":1593,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.20199694Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.20201495Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.212947385Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:711"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.212985675Z","hostname":"buskin","pid":1593,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.99993764s","next_cleanup":"Instant { tv_sec: 10728, tv_nsec: 5838538 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.213011655Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.213071075Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.215492329Z","hostname":"buskin","pid":1593,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.21554193Z","hostname":"buskin","pid":1593,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.215818111Z","hostname":"buskin","pid":1593,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.215906252Z","hostname":"buskin","pid":1593,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV1 {\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV1 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.1.100,\n },\n ],\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.1.174,\n prefix: 24,\n },\n ),\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.216186364Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:979"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.216192324Z","hostname":"buskin","pid":1593,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.216234844Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.216279154Z","hostname":"buskin","pid":1593,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.216317954Z","hostname":"buskin","pid":1593,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.216322504Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1039"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.216346634Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.216358645Z","hostname":"buskin","pid":1593,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.216383375Z","hostname":"buskin","pid":1593,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.216388595Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1080"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.216391715Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.216397035Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.216419115Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.216432355Z","hostname":"buskin","pid":1593,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.217097999Z","hostname":"buskin","pid":1593,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.21725211Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.21727303Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1113"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.21730637Z","hostname":"buskin","pid":1593,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.21731322Z","hostname":"buskin","pid":1593,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.217457401Z","hostname":"buskin","pid":1593,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.217598422Z","hostname":"buskin","pid":1593,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.217635092Z","hostname":"buskin","pid":1593,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 411336b5-dbaf-48b1-93b3-a95d20f0f732, rack_id: 76e0fa5a-5dcb-4021-ab9c-640c3670ef01, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.217857143Z","hostname":"buskin","pid":1593,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:49602","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 411336b5-dbaf-48b1-93b3-a95d20f0f732, rack_id: 76e0fa5a-5dcb-4021-ab9c-640c3670ef01, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.217940884Z","hostname":"buskin","pid":1593,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.217947274Z","hostname":"buskin","pid":1593,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.217975174Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/server.rs:44"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.217979434Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.217982824Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:530"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.217986624Z","hostname":"buskin","pid":1593,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.218007174Z","hostname":"buskin","pid":1593,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"No ledger in /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.218024594Z","hostname":"buskin","pid":1593,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.218038044Z","hostname":"buskin","pid":1593,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.218043384Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:519"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.218046875Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:547"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.220789371Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3849"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.228238504Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.237564519Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.249202998Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.259355017Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.268061598Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.276362007Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.284617186Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.293591309Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.302494721Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.302527271Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:558"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.302531321Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.302535161Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:370"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.302565421Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.302572261Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:449","result":"DisksManagementResult { status: [] }"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.318964808Z","hostname":"buskin","pid":1593,"sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:352"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.319029348Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.319054038Z","hostname":"buskin","pid":1593,"sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:364"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.319107909Z","hostname":"buskin","pid":1593,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 111.011805ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.321911625Z","hostname":"buskin","pid":1593,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1d58111b-934e-4b49-bf27-837dd523bcfd\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:32:10 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1d58111b-934e-4b49-bf27-837dd523bcfd\" }"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.323532035Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"attempting to create encrypted zvol: name=\"oxi_80ce766d-f6eb-440d-b6a8-ad5a1011c02b/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.331772363Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.341002117Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"successfully created encrypted zvol: name=\"oxi_80ce766d-f6eb-440d-b6a8-ad5a1011c02b/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.354374256Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_80ce766d-f6eb-440d-b6a8-ad5a1011c02b/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.354426106Z","hostname":"buskin","pid":1593,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.359184474Z","hostname":"buskin","pid":1593,"sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:379"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.359196274Z","hostname":"buskin","pid":1593,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_80ce766d-f6eb-440d-b6a8-ad5a1011c02b/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.359205284Z","hostname":"buskin","pid":1593,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.428038849Z","hostname":"buskin","pid":1593,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.434125175Z","hostname":"buskin","pid":1593,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 105.831014ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.434745569Z","hostname":"buskin","pid":1593,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"46f64bb9-918f-4910-a8a1-f0b730294684\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:32:10 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"46f64bb9-918f-4910-a8a1-f0b730294684\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.540834672Z","hostname":"buskin","pid":1593,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 215.908375ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.541177494Z","hostname":"buskin","pid":1593,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b1bc65e8-2181-420e-85ae-980cea2dc6af\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:32:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b1bc65e8-2181-420e-85ae-980cea2dc6af\" }"} {"msg":"Mounting oxi_80ce766d-f6eb-440d-b6a8-ad5a1011c02b/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.645714639Z","hostname":"buskin","pid":1593,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.657921901Z","hostname":"buskin","pid":1593,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.657996871Z","hostname":"buskin","pid":1593,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.658036351Z","hostname":"buskin","pid":1593,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.658058801Z","hostname":"buskin","pid":1593,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.758545112Z","hostname":"buskin","pid":1593,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 200.194525ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.759357307Z","hostname":"buskin","pid":1593,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c4231ce1-42a7-4b4f-be12-0cb0c6c424ab\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:32:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c4231ce1-42a7-4b4f-be12-0cb0c6c424ab\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.960363108Z","hostname":"buskin","pid":1593,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 185.930583ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:11.961018942Z","hostname":"buskin","pid":1593,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"826ed309-c41d-4555-8190-782c3bab45e9\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:32:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"826ed309-c41d-4555-8190-782c3bab45e9\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.148247922Z","hostname":"buskin","pid":1593,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 310.316624ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.148886936Z","hostname":"buskin","pid":1593,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4dc0183a-f61e-4c0c-9f45-6900db204a5a\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:32:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4dc0183a-f61e-4c0c-9f45-6900db204a5a\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.404448498Z","hostname":"buskin","pid":1593,"sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.405010942Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:34994","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.405034472Z","hostname":"buskin","pid":1593,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.405116462Z","hostname":"buskin","pid":1593,"file":"oximeter/producer/src/lib.rs:284","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"oximeter/producer/src/lib.rs:284","interval":"30s","address":"[fd00:1122:3344:101::1]:34994","producer_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","route":"//411336b5-dbaf-48b1-93b3-a95d20f0f732"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:12.411050367Z","hostname":"buskin","pid":1593,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"176.285704ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.450893471Z","hostname":"buskin","pid":1593,"component":"VmmReservoirManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.450940232Z","hostname":"buskin","pid":1593,"component":"VmmReservoirManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.451035212Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:993","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.451045942Z","hostname":"buskin","pid":1593,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.451753707Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.451781827Z","hostname":"buskin","pid":1593,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.451809077Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.451818107Z","hostname":"buskin","pid":1593,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.452022958Z","hostname":"buskin","pid":1593,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.452183429Z","hostname":"buskin","pid":1593,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.452209289Z","hostname":"buskin","pid":1593,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.1.100 }], addresses: [V4(Ipv4Network { addr: 10.151.1.174, prefix: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.452216209Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.452232519Z","hostname":"buskin","pid":1593,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:49602","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.628696127Z","hostname":"buskin","pid":1593,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.628758427Z","hostname":"buskin","pid":1593,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.628816317Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.628927038Z","hostname":"buskin","pid":1593,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.628958218Z","hostname":"buskin","pid":1593,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.629000989Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.629014169Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.62923638Z","hostname":"buskin","pid":1593,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:12.630149485Z","hostname":"buskin","pid":1593,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"585.568427ms"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/411336b5-dbaf-48b1-93b3-a95d20f0f732): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:12.634565551Z","hostname":"buskin","pid":1593,"component":"NexusNotifierTask","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.661182908Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:50126"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.661428479Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.66153824Z","hostname":"buskin","pid":1593,"uri":"/inventory","method":"GET","req_id":"c1bd27ec-071b-43fb-ace3-bf6c50dc6e3b","remote_addr":"[fd00:1122:3344:101::1]:50126","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":224,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.69215732Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:53026"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.69225218Z","hostname":"buskin","pid":1593,"uri":"/sled-role","method":"GET","req_id":"2f756672-8848-48ed-b081-79df3e4a7c69","remote_addr":"[fd00:1122:3344:101::1]:53026","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":33,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.693649138Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.693681429Z","hostname":"buskin","pid":1593,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.6939266Z","hostname":"buskin","pid":1593,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.694123751Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:815"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: 593fb985-7780-4b02-bf28-d1cdda078929, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, pool_id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool) }, OmicronPhysicalDiskConfig { id: 56c6ff11-976b-414f-9ee5-247de31e94d7, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, pool_id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool) }, OmicronPhysicalDiskConfig { id: 5d9df6f3-acd0-4b34-9a89-678f436a2e72, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, pool_id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool) }, OmicronPhysicalDiskConfig { id: 60318e90-6e2f-49c0-ac8a-bce5d70c9296, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, pool_id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool) }, OmicronPhysicalDiskConfig { id: be1c0728-b856-4b17-885b-3552285d237a, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, pool_id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool) }, OmicronPhysicalDiskConfig { id: f9a17636-0bdc-4226-90b5-bb7633d77062, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, pool_id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool) }, OmicronPhysicalDiskConfig { id: 40f4ad6e-012f-43b5-bbc3-2175572d8df3, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, pool_id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool) }, OmicronPhysicalDiskConfig { id: c6fe1b14-2e40-4c3f-881d-80d18d9669a7, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, pool_id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool) }, OmicronPhysicalDiskConfig { id: 3611d43e-cd95-4f9f-a0b7-065b796797ff, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, pool_id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.725603146Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:346"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.725766347Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:48070"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, id: 593fb985-7780-4b02-bf28-d1cdda078929, pool_id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, id: 56c6ff11-976b-414f-9ee5-247de31e94d7, pool_id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, id: 5d9df6f3-acd0-4b34-9a89-678f436a2e72, pool_id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, id: 60318e90-6e2f-49c0-ac8a-bce5d70c9296, pool_id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, id: be1c0728-b856-4b17-885b-3552285d237a, pool_id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, id: f9a17636-0bdc-4226-90b5-bb7633d77062, pool_id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, id: 40f4ad6e-012f-43b5-bbc3-2175572d8df3, pool_id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, id: c6fe1b14-2e40-4c3f-881d-80d18d9669a7, pool_id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, id: 3611d43e-cd95-4f9f-a0b7-065b796797ff, pool_id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.726073689Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.726082489Z","hostname":"buskin","pid":1593,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.726091529Z","hostname":"buskin","pid":1593,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.726099079Z","hostname":"buskin","pid":1593,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:681"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.726114719Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.726128489Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.726137169Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.72614434Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.815337994Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:12.815394194Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool)"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:13.225858467Z","hostname":"buskin","pid":1593,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.159154008s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/411336b5-dbaf-48b1-93b3-a95d20f0f732): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-08T03:32:13.573911613Z","hostname":"buskin","pid":1593,"component":"ProbeManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:13.756317045Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:13.921146314Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:13.921317465Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:13.921337285Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:13.921381255Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:13.921408666Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:13.921575557Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:13.921583487Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:14.50569635Z","hostname":"buskin","pid":1593,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.673752662s"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:14.507646721Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:14.692205506Z","hostname":"buskin","pid":1593,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Automatically destroying dataset: oxp_58eb44a6-f8c0-46ec-bcea-52bf3efa8996/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:14.893645581Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/411336b5-dbaf-48b1-93b3-a95d20f0f732): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-08T03:32:15.126139306Z","hostname":"buskin","pid":1593,"component":"ProbeManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/411336b5-dbaf-48b1-93b3-a95d20f0f732): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:15.12667043Z","hostname":"buskin","pid":1593,"component":"NexusNotifierTask","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 64.313882ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:15.504506631Z","hostname":"buskin","pid":1593,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 87.450304ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:16.041667328Z","hostname":"buskin","pid":1593,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/411336b5-dbaf-48b1-93b3-a95d20f0f732): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-08T03:32:16.464171902Z","hostname":"buskin","pid":1593,"component":"ProbeManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:16.464240082Z","hostname":"buskin","pid":1593,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"5.09253006s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 211.253261ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:16.699829477Z","hostname":"buskin","pid":1593,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/411336b5-dbaf-48b1-93b3-a95d20f0f732): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:16.867818715Z","hostname":"buskin","pid":1593,"component":"NexusNotifierTask","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 281.757535ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:17.538794309Z","hostname":"buskin","pid":1593,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/411336b5-dbaf-48b1-93b3-a95d20f0f732): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-08T03:32:17.541448985Z","hostname":"buskin","pid":1593,"component":"ProbeManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_58eb44a6-f8c0-46ec-bcea-52bf3efa8996","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:17.790200837Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:18.230471064Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:18.230544965Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:18.230551825Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:18.230556875Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:18.230577145Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 425.781846ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:18.461954695Z","hostname":"buskin","pid":1593,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:18.470060733Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:18.470069133Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool)"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/411336b5-dbaf-48b1-93b3-a95d20f0f732): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-08T03:32:18.684490484Z","hostname":"buskin","pid":1593,"component":"ProbeManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/411336b5-dbaf-48b1-93b3-a95d20f0f732): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:18.708151293Z","hostname":"buskin","pid":1593,"component":"NexusNotifierTask","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.386795239s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:19.358904968Z","hostname":"buskin","pid":1593,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/411336b5-dbaf-48b1-93b3-a95d20f0f732): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-08T03:32:19.834953196Z","hostname":"buskin","pid":1593,"component":"ProbeManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:19.849840944Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:20.077600242Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:20.077643992Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:20.077658392Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:20.077668562Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:20.077678462Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:20.077800783Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:20.077807173Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:21.061049182Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_888a035b-7ecc-4eff-a005-8001c3d3ccff/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:21.537283882Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/411336b5-dbaf-48b1-93b3-a95d20f0f732): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:22.0643665Z","hostname":"buskin","pid":1593,"component":"NexusNotifierTask","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/411336b5-dbaf-48b1-93b3-a95d20f0f732): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-08T03:32:22.064785123Z","hostname":"buskin","pid":1593,"component":"ProbeManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:22.065405976Z","hostname":"buskin","pid":1593,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"10.119052907s"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:22.803611116Z","hostname":"buskin","pid":1593,"component":"VmmReservoirManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/411336b5-dbaf-48b1-93b3-a95d20f0f732): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:22.810034624Z","hostname":"buskin","pid":1593,"component":"NexusNotifierTask","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Looking for unencrypted datasets in oxp_888a035b-7ecc-4eff-a005-8001c3d3ccff","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:22.908202771Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:22.916061467Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:22.916101707Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:22.916109747Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:22.916118607Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:22.916156748Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:22.925538873Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:22.925575653Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.018947421Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.026886308Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.026924268Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.026942628Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.026955028Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.026969068Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.027119799Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.027127509Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.062627988Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/411336b5-dbaf-48b1-93b3-a95d20f0f732): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-08T03:32:23.067523037Z","hostname":"buskin","pid":1593,"component":"ProbeManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Automatically destroying dataset: oxp_1afb2a46-5b47-46d7-9926-14b7e21c6407/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.070929387Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_1afb2a46-5b47-46d7-9926-14b7e21c6407","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.192978234Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.2006944Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.20072544Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.20073311Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.20074002Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.20076066Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.210848969Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.21087161Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.301335121Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.308896916Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.308942546Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.308947966Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.308986076Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.308994886Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.309137327Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.309142887Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.346070234Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_7c0211ed-cba6-48d8-8b63-a5f21383d980/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.353700919Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_7c0211ed-cba6-48d8-8b63-a5f21383d980","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.466961905Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.474252328Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.474284978Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.474291148Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.474298078Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.474302388Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.483899165Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.483927345Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.579354056Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.587225752Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.587269082Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.587275352Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.587378493Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.587403233Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.587525644Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.587531564Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.623488635Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_fc47eb1d-9a62-49dd-845d-82b29d12e969/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.631403922Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_fc47eb1d-9a62-49dd-845d-82b29d12e969","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.753873632Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.761967679Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.762008399Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.7620158Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.7620258Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.76203189Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.771345674Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.771376545Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.86410192Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.872762321Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.872794761Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.872800841Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.872808051Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.872816431Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.872948742Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.872955802Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.910596963Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_62a3be27-a8c4-4992-8657-cb4aacfdb8ab/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:23.91860581Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_62a3be27-a8c4-4992-8657-cb4aacfdb8ab","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.043668575Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.051672133Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.051709703Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.051734003Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.051740413Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.051745583Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.060975367Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.061008027Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.156937421Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.16526978Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.16530169Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.165309441Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.165334081Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.165346651Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.165494862Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.165502432Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/411336b5-dbaf-48b1-93b3-a95d20f0f732): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-08T03:32:24.169766137Z","hostname":"buskin","pid":1593,"component":"ProbeManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.202385388Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_9edc3ed7-0367-4fc4-87ff-888719d0b682/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.21111557Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.331299036Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:2303"} {"msg":"Looking for unencrypted datasets in oxp_9edc3ed7-0367-4fc4-87ff-888719d0b682","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.33539373Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.344048021Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.344090521Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.344101641Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.344108242Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.344115382Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.354366972Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.354402102Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.45261707Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.46118138Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.46124031Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.46124908Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.46129653Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.461308041Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.461471951Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.461516522Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.497158361Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_ed53811e-9a91-4270-bd15-bcb96ef98de1/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.50545512Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_ed53811e-9a91-4270-bd15-bcb96ef98de1","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.627405577Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.635363424Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.635407884Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.635433834Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.635450794Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.635458414Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.645814235Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.645854146Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.739521616Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.747783505Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.747819385Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.747826975Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.747847485Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.747857435Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.747998926Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.748013646Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.782496329Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.790097534Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/411336b5-dbaf-48b1-93b3-a95d20f0f732): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:24.810202282Z","hostname":"buskin","pid":1593,"component":"NexusNotifierTask","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Looking for unencrypted datasets in oxp_5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.909806967Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.918246357Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.918285117Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Writing ledger to /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.918332727Z","hostname":"buskin","pid":1593,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.918477468Z","hostname":"buskin","pid":1593,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 80ce766d-f6eb-440d-b6a8-ad5a1011c02b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: b3bd066e-6f80-4f80-bc0d-64ec35b22d85 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Writing ledger to /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.918656699Z","hostname":"buskin","pid":1593,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.918983091Z","hostname":"buskin","pid":1593,"uri":"/omicron-physical-disks","method":"PUT","req_id":"0e147a62-88ed-4be6-bda9-236f886c9e89","remote_addr":"[fd00:1122:3344:101::1]:48070","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":12193121,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.95112764Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.951278921Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:36908"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.951561973Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:24.951570493Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:25.003284356Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:25.003330426Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:25.052274194Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:25.052321324Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:25.102392709Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:25.102434409Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:25.102440599Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:25.102489699Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:25.145634263Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:25.145677873Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:25.145684853Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/411336b5-dbaf-48b1-93b3-a95d20f0f732): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-08T03:32:25.175006305Z","hostname":"buskin","pid":1593,"component":"ProbeManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_dd2b8493-b48e-4834-ad8e-a53d21815a61","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:25.197790099Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:25.201883174Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:25.205448495Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_dd2b8493-b48e-4834-ad8e-a53d21815a61","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:25.217958988Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:25.222947917Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:25.227272243Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/411336b5-dbaf-48b1-93b3-a95d20f0f732): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-08T03:32:26.177899321Z","hostname":"buskin","pid":1593,"component":"ProbeManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Profile for oxz_internal_dns_9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:26.714174373Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:26.714261124Z","hostname":"buskin","pid":1593,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:26.714542286Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_internal_dns_dd2b8493-b48e-4834-ad8e-a53d21815a61:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:27.200111929Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:27.680715595Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:27.680787695Z","hostname":"buskin","pid":1593,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703\") failed: Property not found. retry in 60.856651ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:27.687739346Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:27.687805806Z","hostname":"buskin","pid":1593,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:27.687829867Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_dd2b8493-b48e-4834-ad8e-a53d21815a61","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:27.691617529Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703\") failed: Property not found. retry in 116.694476ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:27.758993255Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7\") failed: Property not found. retry in 40.982059ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:27.814700242Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_dd2b8493-b48e-4834-ad8e-a53d21815a61\") failed: Property not found. retry in 56.874572ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:27.820271565Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_dd2b8493-b48e-4834-ad8e-a53d21815a61","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7\") failed: Property not found. retry in 75.468512ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:27.864772127Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_dd2b8493-b48e-4834-ad8e-a53d21815a61\") failed: Property not found. retry in 135.233147ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:27.979444731Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_dd2b8493-b48e-4834-ad8e-a53d21815a61","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703\") failed: Property not found. retry in 274.934785ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:27.995551585Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7\") failed: Property not found. retry in 171.198627ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:28.010315832Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_dd2b8493-b48e-4834-ad8e-a53d21815a61\") failed: Property not found. retry in 135.241013ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:28.126594496Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_dd2b8493-b48e-4834-ad8e-a53d21815a61","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7\") failed: Property not found. retry in 552.283476ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:28.198517879Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_dd2b8493-b48e-4834-ad8e-a53d21815a61\") failed: Property not found. retry in 403.450393ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:28.278191047Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_dd2b8493-b48e-4834-ad8e-a53d21815a61","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703\") failed: Property not found. retry in 204.1125ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:28.293579568Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703\") failed: Property not found. retry in 484.697024ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:28.507470465Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_dd2b8493-b48e-4834-ad8e-a53d21815a61\") failed: Property not found. retry in 1.179763166s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:28.690677532Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_dd2b8493-b48e-4834-ad8e-a53d21815a61","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7\") failed: Property not found. retry in 490.325978ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:28.760188401Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703\") failed: Property not found. retry in 1.089869843s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:29.00350466Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7\") failed: Property not found. retry in 995.485073ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:29.262267091Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_dd2b8493-b48e-4834-ad8e-a53d21815a61\") failed: Property not found. retry in 1.015291816s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:29.880430496Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_dd2b8493-b48e-4834-ad8e-a53d21815a61","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703\") failed: Property not found. retry in 535.708217ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:30.102218589Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7\") failed: Property not found. retry in 760.003307ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:30.266109862Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:9fb2::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:30.37293069Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:2320"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:30.37294515Z","hostname":"buskin","pid":1593,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"wait for service Some(\"oxz_internal_dns_9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703\") failed: Property not found. retry in 1.497719789s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:30.645764434Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_dd2b8493-b48e-4834-ad8e-a53d21815a61\") failed: Property not found. retry in 1.220352103s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:30.904742687Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_dd2b8493-b48e-4834-ad8e-a53d21815a61","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe66:5864","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:30.918265156Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:2327"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:30.923820199Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:2388"} {"msg":"wait for service Some(\"oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7\") failed: Property not found. retry in 624.987175ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:31.035505294Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:31.049476967Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:2573"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:31.298217499Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:2737"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:31.490259508Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:2452"} {"msg":"wait for service Some(\"oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7\") failed: Property not found. retry in 795.478374ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:31.668988779Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:31.808286008Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:2487"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:32.018754925Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:2787"} {"msg":"wait for service Some(\"oxz_internal_dns_dd2b8493-b48e-4834-ad8e-a53d21815a61\") failed: Property not found. retry in 922.731924ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:32.134695187Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_dd2b8493-b48e-4834-ad8e-a53d21815a61","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703\") failed: Property not found. retry in 959.781397ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:32.151523606Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:32.200615284Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:2814"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:32.426032539Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:2681"} {"msg":"wait for service Some(\"oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7\") failed: Property not found. retry in 869.286182ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:32.474678335Z","hostname":"buskin","pid":1593,"zone":"oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:32.630085719Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:2481"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:32.641911118Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3873","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:32.641947538Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3889"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:32.641955489Z","hostname":"buskin","pid":1593,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.207980885Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3897"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.213556878Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3953"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.347707927Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:4015"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.364390455Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_dd2b8493-b48e-4834-ad8e-a53d21815a61"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.364455485Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_cc505437-f47e-4e85-84e3-c10ec252eef7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.364465305Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703"} {"msg":"Writing ledger to /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.364474865Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.364791287Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.365028509Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"0e5f39fe-221e-4371-a8ae-224bf9ea0bdd","remote_addr":"[fd00:1122:3344:101::1]:36908","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8413682,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.367023711Z","hostname":"buskin","pid":1593,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:547"} {"msg":"failed to write DNS configuration (will retry in 110.134897ms)","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:33.424356928Z","hostname":"buskin","pid":1593,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:567","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.446992911Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3996","rack_subnet":"fd00:1122:3344::"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.469641224Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:4043"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.556180383Z","hostname":"buskin","pid":1593,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:547"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.567223897Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:4070"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.619908837Z","hostname":"buskin","pid":1593,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:547"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.684936969Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:583"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.68497803Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.687948857Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:364"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.748952916Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.750655006Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:412"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.750907647Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.750912797Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.806918657Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.807033367Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:38630"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.807404609Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.80741348Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.812732351Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:589"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.812835321Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:627"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.813225864Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:630","uptime":"2"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.813234424Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:434","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.828800665Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.828863295Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.846173477Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.846207217Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.863067387Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.863107467Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.863288738Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.863400749Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.87728457Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.909948352Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.910022233Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.910037703Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.910046673Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.910052503Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.910071613Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.910079243Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.910086083Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.910092823Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.910103223Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.910109843Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.910116453Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.910123023Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.910129633Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.910136453Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.910143604Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.910150113Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.910168173Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.910173494Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.910178494Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.910183274Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.910188174Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.910193084Z","hostname":"buskin","pid":1593,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.919212707Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.920964027Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.920970437Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.965476259Z","hostname":"buskin","pid":1593,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 254, 79]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.965532369Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.966160533Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.966174313Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.966489155Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.968027024Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:33.991014559Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:34.010968996Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:34.610093578Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:34.610260959Z","hostname":"buskin","pid":1593,"zone":"oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\") failed: Property not found. retry in 25.64676ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:34.73969699Z","hostname":"buskin","pid":1593,"zone":"oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\") failed: Property not found. retry in 95.17991ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:34.771531987Z","hostname":"buskin","pid":1593,"zone":"oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\") failed: Property not found. retry in 139.26904ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:34.881212232Z","hostname":"buskin","pid":1593,"zone":"oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\") failed: Property not found. retry in 301.398524ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:35.035173537Z","hostname":"buskin","pid":1593,"zone":"oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\") failed: Property not found. retry in 441.194036ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:35.345214589Z","hostname":"buskin","pid":1593,"zone":"oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\") failed: Property not found. retry in 810.900031ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:35.796572542Z","hostname":"buskin","pid":1593,"zone":"oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\") failed: Property not found. retry in 1.176822756s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:36.616025449Z","hostname":"buskin","pid":1593,"zone":"oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:37.090987681Z","hostname":"buskin","pid":1593,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45359"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:37.091135032Z","hostname":"buskin","pid":1593,"uri":"/baseboard","method":"GET","req_id":"2a15ce44-33a1-4a5f-85f9-2903497ec2ac","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45359","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":75,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\") failed: Property not found. retry in 1.024015016s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:37.802519494Z","hostname":"buskin","pid":1593,"zone":"oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\") failed: Property not found. retry in 994.549401ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:38.834842672Z","hostname":"buskin","pid":1593,"zone":"oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:39.852524664Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed"} {"msg":"Writing ledger to /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:39.852593825Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:39.852892477Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:39.853144528Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"6e88f0bb-0cc7-40be-aa1c-bf67b259fb30","remote_addr":"[fd00:1122:3344:101::1]:38630","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":6045937,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:39.855262811Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:624"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:39.884541153Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:39.884681404Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:60455"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:39.894372261Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:39.894496951Z","hostname":"buskin","pid":1593,"uri":"/timesync","method":"GET","req_id":"1d2884c7-ff3d-4e3e-bf37-41a65479c7be","remote_addr":"[fd00:1122:3344:101::1]:60455","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9668,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:39.897118907Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:40.060315125Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:40.060417056Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:33421"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:44.069292481Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:44.069360401Z","hostname":"buskin","pid":1593,"uri":"/timesync","method":"GET","req_id":"53a8cfb9-a554-4453-8ae3-9ee72eebda6d","remote_addr":"[fd00:1122:3344:101::1]:33421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008860,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:44.072648821Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:44.281848251Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:44.281961431Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:59919"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:44.290867634Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:44.291034935Z","hostname":"buskin","pid":1593,"uri":"/timesync","method":"GET","req_id":"e9494a15-784d-4545-b6ec-4cd9bd2403ab","remote_addr":"[fd00:1122:3344:101::1]:59919","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8903,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:44.294222163Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:44.528005558Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:44.528129539Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:50012"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_6d4f2f0d-0048-43a0-b797-4120ab6bb8ed': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:44.535766183Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:44.535914544Z","hostname":"buskin","pid":1593,"uri":"/timesync","method":"GET","req_id":"2bf2f528-ee4b-4315-a692-ca38476c29ad","remote_addr":"[fd00:1122:3344:101::1]:50012","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":7682,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:44.537834596Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:44.761257949Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:44.76136107Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:36786"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:48.770534206Z","hostname":"buskin","pid":1593,"uri":"/timesync","method":"GET","req_id":"781b0ab8-4226-4ec0-908c-ec3a30e44e8d","remote_addr":"[fd00:1122:3344:101::1]:36786","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":4009056,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1715139163.6657124, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:48.773257562Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:48.773267373Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.046147426Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.046332867Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:35500"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.05528613Z","hostname":"buskin","pid":1593,"uri":"/timesync","method":"GET","req_id":"31e09e82-0745-4f71-b712-dcc9b2067932","remote_addr":"[fd00:1122:3344:101::1]:35500","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8883,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1715139163.6657124, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.059226853Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:49.059241123Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.316504465Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.316632596Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:46208"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.324970865Z","hostname":"buskin","pid":1593,"uri":"/timesync","method":"GET","req_id":"fc2975b5-eccb-441b-b86c-23c2bdf1589c","remote_addr":"[fd00:1122:3344:101::1]:46208","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8168,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1715139163.6657124, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.327203618Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:49.327210948Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.010507382Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.010620142Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:41588"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.014306394Z","hostname":"buskin","pid":1593,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.031661816Z","hostname":"buskin","pid":1593,"uri":"/timesync","method":"GET","req_id":"467221b2-203b-4e8b-910b-e2549133ff20","remote_addr":"[fd00:1122:3344:101::1]:41588","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":20867,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2395651603, ip_addr: 142.202.190.19, stratum: 3, ref_time: 1715139168.8485675, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.033924329Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.033929989Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1192"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.062838039Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.06297712Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:59595"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.063395783Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.063416473Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.084479757Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.084527887Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.103054076Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.103107536Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.120946891Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.120997291Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.174249694Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.174293034Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.226160079Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.226216789Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.277315899Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.27735966Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.329231095Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.329281695Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.380071804Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.380129654Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.380385636Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.380524646Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.461464882Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.461523122Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.461532882Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.461558033Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.461564013Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_8a2bf75b-8c7f-4d8d-9ede-6fc064124d25","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.554455619Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.558617663Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.562577376Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.56651981Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.571103057Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_8a2bf75b-8c7f-4d8d-9ede-6fc064124d25","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.575454342Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.579605517Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.584106783Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.5885772Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:49.593040026Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:51.439692751Z","hostname":"buskin","pid":1593,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"321.822666ms"} {"msg":"Profile for oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:52.851351295Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:52.851486916Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:52.851506716Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:52.854881406Z","hostname":"buskin","pid":1593,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:52.85731912Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:52.896833072Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:52.896938823Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:52.938489277Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:52.938604498Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_8a2bf75b-8c7f-4d8d-9ede-6fc064124d25:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:52.955785219Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:32:52.95590545Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_8a2bf75b-8c7f-4d8d-9ede-6fc064124d25","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d\") failed: Property not found. retry in 58.964782ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:53.004172143Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5\") failed: Property not found. retry in 31.726592ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:53.012637013Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709\") failed: Property not found. retry in 59.69256ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:53.046536822Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5\") failed: Property not found. retry in 114.174639ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:53.055024052Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d\") failed: Property not found. retry in 148.643456ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:53.079310175Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120\") failed: Property not found. retry in 60.637878ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:53.205066314Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8a2bf75b-8c7f-4d8d-9ede-6fc064124d25\") failed: Property not found. retry in 67.39889ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:53.497551324Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_8a2bf75b-8c7f-4d8d-9ede-6fc064124d25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709\") failed: Property not found. retry in 59.036758ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:53.628434683Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5\") failed: Property not found. retry in 216.831802ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:54.002059959Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8a2bf75b-8c7f-4d8d-9ede-6fc064124d25\") failed: Property not found. retry in 62.479001ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:54.052423005Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_8a2bf75b-8c7f-4d8d-9ede-6fc064124d25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d\") failed: Property not found. retry in 178.433108ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:54.302387495Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120\") failed: Property not found. retry in 51.727317ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:54.486604318Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709\") failed: Property not found. retry in 192.595752ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:54.865503665Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5\") failed: Property not found. retry in 324.152317ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:55.182070656Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8a2bf75b-8c7f-4d8d-9ede-6fc064124d25\") failed: Property not found. retry in 256.260677ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:55.36979938Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_8a2bf75b-8c7f-4d8d-9ede-6fc064124d25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d\") failed: Property not found. retry in 366.776872ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:55.426185331Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120\") failed: Property not found. retry in 115.545414ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:55.45142212Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709\") failed: Property not found. retry in 392.330261ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:55.476554587Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5\") failed: Property not found. retry in 565.686086ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:55.519311039Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120\") failed: Property not found. retry in 304.141871ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:55.578010604Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8a2bf75b-8c7f-4d8d-9ede-6fc064124d25\") failed: Property not found. retry in 581.581852ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:55.637244202Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_8a2bf75b-8c7f-4d8d-9ede-6fc064124d25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d\") failed: Property not found. retry in 454.936774ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:55.804404724Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709\") failed: Property not found. retry in 409.317139ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:55.88023247Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120\") failed: Property not found. retry in 876.753889ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:55.892846994Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5\") failed: Property not found. retry in 662.881779ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:56.096112879Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8a2bf75b-8c7f-4d8d-9ede-6fc064124d25\") failed: Property not found. retry in 990.206685ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:56.230551009Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_8a2bf75b-8c7f-4d8d-9ede-6fc064124d25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d\") failed: Property not found. retry in 601.327205ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:56.271640851Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709\") failed: Property not found. retry in 949.078475ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:56.298310507Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5\") failed: Property not found. retry in 817.177909ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:56.769873379Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120\") failed: Property not found. retry in 1.481391195s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:56.778130888Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d\") failed: Property not found. retry in 1.133304032s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:56.88221511Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8a2bf75b-8c7f-4d8d-9ede-6fc064124d25\") failed: Property not found. retry in 1.093465529s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:57.230311436Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_8a2bf75b-8c7f-4d8d-9ede-6fc064124d25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709\") failed: Property not found. retry in 712.256301ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:57.256238578Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5\") failed: Property not found. retry in 512.96652ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:57.597305414Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709\") failed: Property not found. retry in 1.430567558s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:57.981003139Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d\") failed: Property not found. retry in 1.403106239s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:58.026431116Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5\") failed: Property not found. retry in 607.048372ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:58.12229908Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120\") failed: Property not found. retry in 626.387275ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:58.271250775Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8a2bf75b-8c7f-4d8d-9ede-6fc064124d25\") failed: Property not found. retry in 1.247213134s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:58.337504695Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_8a2bf75b-8c7f-4d8d-9ede-6fc064124d25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5\") failed: Property not found. retry in 576.247161ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:58.742933498Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120\") failed: Property not found. retry in 1.363314959s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:58.90655744Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5\") failed: Property not found. retry in 928.944017ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:59.328239639Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709\") failed: Property not found. retry in 980.725079ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:59.419763967Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d\") failed: Property not found. retry in 1.139330304s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:59.43729242Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8a2bf75b-8c7f-4d8d-9ede-6fc064124d25\") failed: Property not found. retry in 940.114116ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:32:59.593050525Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_8a2bf75b-8c7f-4d8d-9ede-6fc064124d25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5\") failed: Property not found. retry in 1.498957952s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:00.26741689Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120\") failed: Property not found. retry in 954.914015ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:00.279633482Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709\") failed: Property not found. retry in 1.259272197s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:00.409973458Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8a2bf75b-8c7f-4d8d-9ede-6fc064124d25\") failed: Property not found. retry in 1.222144485s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:00.542108735Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_8a2bf75b-8c7f-4d8d-9ede-6fc064124d25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d\") failed: Property not found. retry in 1.285319631s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:00.585399369Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120\") failed: Property not found. retry in 916.320822ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:01.246201863Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709\") failed: Property not found. retry in 1.260922985s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:01.68293223Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709\") failed: Property not found. retry in 897.540795ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:02.955051158Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709\") failed: Property not found. retry in 755.785398ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:03.865887092Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709\") failed: Property not found. retry in 1.343746949s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:04.633342984Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709\") failed: Property not found. retry in 1.495535641s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:06.014613724Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:06.810022729Z","hostname":"buskin","pid":1593,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65129"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:06.810454732Z","hostname":"buskin","pid":1593,"uri":"/baseboard","method":"GET","req_id":"1be8e7ed-b1c5-4a8f-9372-c08d59e6cad6","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65129","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":105,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:07.265978859Z","hostname":"buskin","pid":1593,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"348.989711ms"} {"msg":"wait for service Some(\"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709\") failed: Property not found. retry in 1.135795916s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:07.525647176Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709\") failed: Property not found. retry in 1.267229279s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:08.684704778Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709\") failed: Property not found. retry in 941.289995ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:09.961494943Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709\") failed: Property not found. retry in 1.155043812s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:10.91170378Z","hostname":"buskin","pid":1593,"zone":"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:12.088238885Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_8a2bf75b-8c7f-4d8d-9ede-6fc064124d25"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:12.088318426Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_45e11811-5e91-4520-b80a-7f642e885b6d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:12.088347026Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_14c4878d-c5dd-4fc2-8846-aaf3152f4120"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:12.088354256Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_f44a281c-ca8a-47a1-9feb-a5b347543709"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:12.088360836Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_ae06d4ce-795c-4d42-b79d-9db3f61d73a5"} {"msg":"Writing ledger to /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:12.088374736Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:12.089134931Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:12.090081046Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"c9dea3d6-aac0-49e5-b1ea-8e6976f7ebb6","remote_addr":"[fd00:1122:3344:101::1]:59595","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":22558010,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:12.123508773Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:54375"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::5]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:12.152641314Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3437"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:12.334525323Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3455"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.226845203Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3478"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.226952783Z","hostname":"buskin","pid":1593,"uri":"/cockroachdb","method":"POST","req_id":"4ef6808f-d23a-472e-93e1-d050285370f1","remote_addr":"[fd00:1122:3344:101::1]:54375","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":4103334,"response_code":"204"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.260167579Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.260535131Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:64762"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.261529267Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.261537607Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.284754643Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.284803093Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.306082818Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.306140929Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.327773166Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.327817376Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.347763923Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.347808294Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.367039097Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.367088017Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.387455477Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.387621798Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.408089128Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.408135888Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.428165646Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.428212676Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.507873085Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.507929795Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.589300253Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.589353634Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.669195893Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.669283553Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.74529501Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.745348911Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.818748552Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.818802883Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.908119718Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.908174258Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.982471085Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:16.982529245Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.050389034Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.050456124Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.115623128Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.115678198Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.180385988Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.180438649Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.251582927Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.251650707Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.320717803Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.320790004Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.321120236Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.321278197Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:17.433379865Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:17.433460446Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:17.433468366Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:17.433475376Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:17.433483926Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:17.433492256Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:17.433500536Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:17.433507246Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:17.433512526Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:17.433520076Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:17.433527606Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:17.433533246Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:17.433538806Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:17.433546257Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:17.433553526Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:17.433568447Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:17.433572827Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:17.433576607Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:17.433580267Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.433663287Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"0ef94c10-fbf1-42b4-bdfa-d80c3d0f10b0","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized)]","latency_us":1172386,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:17.433910319Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"124.107936ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0ef94c10-fbf1-42b4-bdfa-d80c3d0f10b0\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:33:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0ef94c10-fbf1-42b4-bdfa-d80c3d0f10b0\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.559083415Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.55993254Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.55994326Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.584210962Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.584267403Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.608924508Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.608974968Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.630010312Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.630063802Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.650706102Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.650756133Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.670363088Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.670413398Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.69102965Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.69107258Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.711381599Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.71142941Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.732442563Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.732486873Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.753531827Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.753579407Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.774659571Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.774703031Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.795172152Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.795228322Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.815861213Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.815908494Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.836184253Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.836232353Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.856415482Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.856461162Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.876928723Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.876974733Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.898759591Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.898804981Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.920112516Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.920154746Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.939830182Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.939877193Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.95990299Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.959950571Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.98026424Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.98031414Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.980547912Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:17.980688482Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.078466957Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.078524708Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.078533598Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.078539888Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.078545658Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.078553098Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.078558948Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.078564768Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.078570278Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.078575648Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.078581218Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.078587128Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.078594078Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.078599868Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.078626218Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.078632168Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.078639288Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.078644738Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.078650228Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.078739669Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"6bee72c8-1242-4c47-95c1-c845c726bf43","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized)]","latency_us":519008,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.07892972Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"164.722531ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6bee72c8-1242-4c47-95c1-c845c726bf43\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:33:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6bee72c8-1242-4c47-95c1-c845c726bf43\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.244201331Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.244534093Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.244543033Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.268722896Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.268773876Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.29154607Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.29159809Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.312339272Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.312389643Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.333184685Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.333233805Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.355649507Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.355700797Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.374562978Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.374624048Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.396415117Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.396466867Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.416299593Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.416351764Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.438226062Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.438277123Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.460707704Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.460759475Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.483865071Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.483929821Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.505197766Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.505248026Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.527031674Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.527077065Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.547988528Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.548033848Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.570898302Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.570944492Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.594223199Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.59426888Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.615412544Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.615456404Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.637769005Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.637816216Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.657060619Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.657109319Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.677988602Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.678050222Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.678242923Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.678390884Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.782105444Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.782184684Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.782192974Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.782198804Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.782205004Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.782212054Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.782217665Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.782224885Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.782232254Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.782238025Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.782243505Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.782255735Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.782262735Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.782268155Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.782282905Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.782286945Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.782291055Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.782294825Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.782298695Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.782371065Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"c67478f7-f963-46e3-a888-79b2926dee5d","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized)]","latency_us":538035,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:18.782571256Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"76.98722ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c67478f7-f963-46e3-a888-79b2926dee5d\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:33:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c67478f7-f963-46e3-a888-79b2926dee5d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.860805387Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.861119708Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.861127868Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.88349327Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.88354558Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.908591487Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.908639048Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.931658673Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.931702213Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.951897502Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.951947802Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.97367801Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.97372047Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.993428996Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:18.993482336Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.014043647Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.014086997Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.034704189Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.034742129Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.05362937Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.05366604Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.073662468Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.073707088Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.09612309Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.09616641Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.115760255Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.115805335Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.138116717Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.138165077Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.158133494Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.158176785Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.177387548Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.177427228Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.196099288Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.196137568Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.216725979Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.216767829Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.235379138Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.235428289Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.254259649Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.25430088Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.273160051Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.273205881Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.273440362Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.273576533Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:19.37344656Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:19.373519161Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:19.373528411Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:19.373534701Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:19.373540961Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:19.373549171Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:19.373555551Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:19.373561561Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:19.373567431Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:19.373573081Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:19.373580531Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:19.373584621Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:19.373590271Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:19.373618511Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:19.373624451Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:19.373630211Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:19.373637751Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:19.373643341Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:19.373648731Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.373796322Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"b52c94f3-cd06-4e76-8767-d296c683f35f","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized)]","latency_us":512838,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:19.374014413Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"235.807319ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b52c94f3-cd06-4e76-8767-d296c683f35f\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:33:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b52c94f3-cd06-4e76-8767-d296c683f35f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.611263018Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.611583Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.611593Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.635598441Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.635662672Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.656095731Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.656136371Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.674189717Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.674236598Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.694881119Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.694924039Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.713331578Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.713372948Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.731631575Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.731683015Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.749764092Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.749806622Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.768294811Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.768342271Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.790932294Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.790975194Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.810173117Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.810216367Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.828774456Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.828820306Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.849102616Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.849144396Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.869887798Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.869929438Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.888806839Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.888853069Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.909586691Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.909643021Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.930203293Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.930255033Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.951574328Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.951616148Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.971694406Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.971736697Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.990541567Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:19.990588837Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.011514091Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.011592201Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.011795072Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.011937063Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.107403074Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.107474544Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.107481755Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.107486375Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.107493865Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.107501265Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.107508545Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.107515605Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.107521135Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.107528205Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.107533375Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.107538795Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.107557435Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.107564865Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.107570275Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.107574145Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.107578185Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.107583675Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.107587295Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.107662896Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"af531150-d092-45ca-b74a-4d55561645ad","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized)]","latency_us":496282,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.107853287Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"204.72719ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"af531150-d092-45ca-b74a-4d55561645ad\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:33:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"af531150-d092-45ca-b74a-4d55561645ad\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.313490066Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.314436371Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.314447211Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.339022746Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.339081146Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.361080526Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.361126856Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.380308759Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.380355279Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.398617676Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.398656266Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.417793349Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.417851889Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.436995382Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.437045792Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.45882237Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.45887274Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.480333427Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.480383507Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.501050738Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.501099599Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.520528923Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.520577393Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.5405156Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.540563021Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.56076798Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.56081374Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.579734961Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.579776371Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.598564462Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.598615032Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.618348058Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.618408718Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.637011868Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.637052938Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.655406725Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.655451365Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.675362742Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.675404773Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.694541005Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.694581275Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.713747788Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.713799199Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.71402816Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.714175911Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.806365793Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.806408083Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.806414963Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.806420343Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.806427633Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.806433763Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.806439483Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.806445243Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.806452363Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.806456543Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.806462033Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.806466093Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.806488733Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.806496323Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.806503724Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.806509684Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.806516913Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.806522324Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.806529193Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.806650874Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"242b5fad-1f87-4688-981b-82221cc80276","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized)]","latency_us":493021,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:20.806857095Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"186.208505ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"242b5fad-1f87-4688-981b-82221cc80276\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:33:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"242b5fad-1f87-4688-981b-82221cc80276\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.994240147Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.994532339Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:20.994540949Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.014297865Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.014344835Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.034413023Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.034455873Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.052287958Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.052333019Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.071555562Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.071605022Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.088954924Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.089009204Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.106485427Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.106524177Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.123792099Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.123833379Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.141321842Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.141368442Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.159180327Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.159223697Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.176922941Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.176974041Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.194472354Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.194518114Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.211717286Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.211753246Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.230555146Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.230596757Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.247855018Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.247901518Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.265809874Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.265867774Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.283497908Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.283539288Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.302256128Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.302298068Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.319980532Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.320024972Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.338155519Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.338194709Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.356340186Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.356398056Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.356605997Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.356737678Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:21.4506308Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:21.450709831Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:21.450717311Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:21.450724931Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:21.450728941Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:21.450735881Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:21.450741461Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:21.450748891Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:21.450756301Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:21.450762261Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:21.450768961Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:21.450786051Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:21.450791911Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:21.450799341Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:21.450806421Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:21.450813761Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:21.450817901Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:21.450821601Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:21.450825391Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.450919262Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"7ef614c6-f824-4dc5-9371-d4b411775ad0","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized)]","latency_us":456553,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:21.451136023Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"265.496056ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7ef614c6-f824-4dc5-9371-d4b411775ad0\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:33:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7ef614c6-f824-4dc5-9371-d4b411775ad0\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.717598959Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.717970671Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.717982711Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.742571626Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.742623406Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.7638056Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.763860331Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.784359301Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.784414052Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.804705981Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.804752941Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.825846715Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.825901805Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.846587937Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.846637947Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.866841226Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.866891457Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.886047309Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.886088679Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.907177843Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.907220603Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.92872481Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.92877424Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.949369291Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.949412951Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.969707551Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.969753941Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.991047596Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:21.991095617Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.010688842Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.010739252Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.030305457Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.030358778Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.053224802Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.053273692Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.073132179Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.073180249Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.094585825Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.094636195Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.115009925Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.115060265Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.135029173Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.135092963Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.135332635Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.135494965Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:22.231385019Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:22.23145477Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:22.23146796Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:22.23147719Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:22.23148373Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:22.23149061Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:22.23149715Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:22.23150392Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:22.23151024Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:22.231517Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:22.23152574Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:22.23155388Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:22.23156268Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:22.23156928Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:22.23157725Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:22.23158403Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:22.23159255Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:22.23160053Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:22.231608841Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.231733441Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"629cb140-6e9a-4d6f-a875-0d3476ba7544","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized)]","latency_us":513979,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:22.231951852Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"338.433134ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"629cb140-6e9a-4d6f-a875-0d3476ba7544\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:33:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"629cb140-6e9a-4d6f-a875-0d3476ba7544\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.571517549Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.571870591Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.571880331Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.593278527Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.593322877Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.611805396Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.611848546Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.629748381Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.629791471Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.647485475Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.647532756Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:22.665622401Z","hostname":"buskin","pid":1593,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"581.374779ms"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.666050264Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.666070174Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.684059669Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.68410989Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.701571783Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.701613383Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.719226916Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.719273367Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.736993391Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.737047851Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.755930692Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.755971302Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.774396971Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.774439861Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.793617884Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.793659334Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.811483589Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.811525199Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.831943169Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.831984639Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.85082078Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.85087162Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.87120132Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.87124269Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.892897547Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.892941988Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.913597039Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.913641669Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.931573805Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.931621235Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.949209548Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.949265639Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.94949284Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:22.949631901Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:23.043326672Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:23.043380902Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:23.043389212Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:23.043395222Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:23.043402602Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:23.043408522Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:23.043415942Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:23.043421862Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:23.043439522Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:23.043447162Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:23.043452872Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:23.043458462Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:23.043463762Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:23.043470893Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:23.043476202Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:23.043481862Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:23.043488973Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:23.043495923Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:23.043501603Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.043642434Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"e0fdc4ef-bb40-4421-acea-f3b2ce7d5e5a","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized)]","latency_us":471991,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:23.043848405Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"460.769219ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e0fdc4ef-bb40-4421-acea-f3b2ce7d5e5a\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:33:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e0fdc4ef-bb40-4421-acea-f3b2ce7d5e5a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.50565873Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.505987221Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.505998152Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.532275586Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.532333907Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.553472631Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.553528681Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.574750476Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.574807446Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.59415887Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.59422369Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.613828115Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.613890756Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.632529085Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.632582946Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.654130032Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.654175392Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.671651685Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.671697485Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.692432847Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.692486687Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.712845737Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.712895727Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.73382226Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.73386501Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.755714799Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.755761899Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.776138079Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.776185799Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.797241663Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.797290263Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.819157102Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.819211662Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.842154187Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.842206247Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.864102926Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.864155916Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.885548362Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.885609512Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.904656075Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.904705695Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.923348974Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.923421025Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.923620696Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:23.923762027Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.019444099Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.01950928Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.01951777Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.01952548Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.01953178Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.01953594Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.01954012Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.01956585Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.01957186Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.01957887Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.01958571Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.01959173Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.01959874Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.01960587Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.01961181Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.01961718Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.01962411Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.01962953Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.01963657Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.019744901Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"df64001f-51c4-4358-a96b-ed96e37546ca","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized)]","latency_us":513939,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.019933962Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"390.768995ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"df64001f-51c4-4358-a96b-ed96e37546ca\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:33:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"df64001f-51c4-4358-a96b-ed96e37546ca\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.412583101Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.412994923Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.413003333Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.435598446Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.435642516Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.455438632Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.455488873Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.475292559Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.475328239Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.49421521Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.494272161Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.513566094Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.513608604Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.532686436Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.532722347Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.551481667Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.551519107Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.570300398Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.570330768Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.595374385Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.595425565Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.619631448Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.619666428Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.647664583Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.647699853Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.671601472Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.671640272Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.696999462Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.697042002Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.721500926Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.721537606Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.745619107Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.745664948Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.769580148Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.769618908Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.794068682Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.794111303Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.820315377Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.820369507Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.845635495Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.845677006Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.87030666Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.870350001Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.870569842Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.870722423Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.981779916Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.981846616Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.981854016Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.981861056Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.981865126Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.981869106Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.981873396Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.981888267Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.981892696Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.981896696Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.981900676Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.981907127Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.982015707Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.982025367Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.982031287Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.982038327Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.982042167Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.982047777Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.982053088Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:24.982230198Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"d8b98227-5fa2-4251-a356-c45aa57c6482","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized)]","latency_us":569434,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:24.98247067Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"374.092592ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d8b98227-5fa2-4251-a356-c45aa57c6482\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:33:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d8b98227-5fa2-4251-a356-c45aa57c6482\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.356981141Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.357306434Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.357320274Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.378975061Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.379028741Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.39919194Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.39924287Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.417732519Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.417778669Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.437759036Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.437822027Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.456240995Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.456284555Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.47406818Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.47411622Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.493153682Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.493204133Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.513446992Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.513492082Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.532436863Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.532485093Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.550960362Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.551006452Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.571280772Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.571331462Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.590717346Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.590776286Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.608895382Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.608955513Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.627318941Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.627365241Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.647164867Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.647203788Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.666011988Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.666056248Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.684445996Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.684502146Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.702603243Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.702648333Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.720431117Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.720485018Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.738180712Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.738243342Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.738469374Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.738653944Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:25.832463306Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:25.832507566Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:25.832515806Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:25.832523256Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:25.832530036Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:25.832537736Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:25.832559516Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:25.832564127Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:25.832571547Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:25.832577147Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:25.832582707Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:25.832589997Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:25.832597027Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:25.832602937Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:25.832609927Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:25.832617227Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:25.832622847Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:25.832629697Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:25.832635087Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:25.832763538Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"2e7f830a-cf50-42c0-9d7b-5cec7ab409e4","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized)]","latency_us":475653,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:25.83305579Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"876.437488ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2e7f830a-cf50-42c0-9d7b-5cec7ab409e4\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:33:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2e7f830a-cf50-42c0-9d7b-5cec7ab409e4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.710523318Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.71083944Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.71084836Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.733246342Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.733298842Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.7551426Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.75518998Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.77546955Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.77552214Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.794830624Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.794873524Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.812728429Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.812772529Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.833447391Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.833493341Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.851831999Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.851876619Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.872777602Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.872821992Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.89118938Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.891248831Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.913693822Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.913747043Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.934542975Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.934595405Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.954728524Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.954776384Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.975987218Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.976072199Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.995780785Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:26.995832365Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:27.014487755Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:27.014533355Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:27.033970979Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:27.03401786Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:27.055295135Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:27.055338815Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:27.074350107Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:27.074400107Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:27.092293972Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:27.092340843Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:27.111033623Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:27.111091043Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:27.111291814Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:27.111450415Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:27.203679547Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:27.203752657Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:27.203760668Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:27.203766348Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:27.203772008Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:27.203790008Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:27.203795968Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:27.203802888Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:27.203808988Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:27.203814578Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:27.203821578Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:27.203826978Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:27.203834308Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:27.203841488Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:27.203846798Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:27.203853698Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:27.203859018Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:27.203864188Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:27.203871388Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:27.203992409Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"1494a8fb-a3a7-49b7-8bf9-79c81be99913","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized)]","latency_us":493341,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:27.20420565Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.16875311s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1494a8fb-a3a7-49b7-8bf9-79c81be99913\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:33:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1494a8fb-a3a7-49b7-8bf9-79c81be99913\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.374905692Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.375452615Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.375470465Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.397295073Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.397351224Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.417874194Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.418093006Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.43594036Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.435986851Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.453194972Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.453234172Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.470549214Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.470587694Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.488027497Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.488068447Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.50556931Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.50561615Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.524946564Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.524998804Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.543268351Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.543308922Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.561118276Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.561182447Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.579839457Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.579878817Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.598043704Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.598080464Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.615763788Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.615815848Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.633299911Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.633341491Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.651040894Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.651081584Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.668774159Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.668820079Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.688315623Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.688352154Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.70808226Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.70811839Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.726632989Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.726677229Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.746729977Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.746774147Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.746998328Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.747132939Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:28.841760065Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:28.841827416Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:28.841837136Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:28.841845926Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:28.841852476Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:28.841884586Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:28.841892916Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:28.841897606Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:28.841902446Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:28.841908656Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:28.841913216Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:28.841917526Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:28.841922166Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:28.841929557Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:28.841933857Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:28.841940346Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:28.841945077Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:28.841953097Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:28.841959287Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:28.842015007Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"4a1e627f-85a6-45ee-af4f-488c2adc5ca6","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized)]","latency_us":466867,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:28.842238918Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.069011227s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4a1e627f-85a6-45ee-af4f-488c2adc5ca6\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:33:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4a1e627f-85a6-45ee-af4f-488c2adc5ca6\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:29.912204908Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:29.913119003Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:29.913138423Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:29.935401094Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:29.935455064Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:29.957718035Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:29.957772625Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:29.9773187Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:29.977366921Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:29.996339352Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:29.996385423Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.015542415Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.015587975Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.034805228Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.034847879Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.053995861Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.054041042Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.07422283Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.07427809Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.094049937Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.094093997Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.11334367Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.11339378Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.133321208Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.133364038Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.153380286Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.153429596Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.172366477Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.172409717Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.192165263Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.192215114Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.211371556Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.211423477Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.229281822Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.229323582Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.248304584Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.248348184Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.267313655Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.267359645Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.285348811Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.285393862Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.304223962Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.304286563Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.304477394Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.304602655Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:30.39740385Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:30.39746272Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:30.39747035Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:30.397475881Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:30.397489081Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:30.39749619Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:30.397501901Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:30.397508471Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:30.397514011Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:30.397521081Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:30.397526431Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:30.397533591Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:30.397540651Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:30.397545991Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:30.397551721Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:30.397558901Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:30.397564291Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:30.397570851Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:30.397574671Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:30.397670902Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"faa0beed-3116-4e43-af5c-e7f07c990390","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized)]","latency_us":485151,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:30.397882243Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.543455881s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"faa0beed-3116-4e43-af5c-e7f07c990390\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:33:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"faa0beed-3116-4e43-af5c-e7f07c990390\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:31.942765644Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:31.943083676Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:31.943094136Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:31.966047151Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:31.966098271Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:31.985630766Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:31.985683757Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.006442689Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.006502749Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.026482297Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.026530137Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.045034966Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.045080066Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.063021991Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.063071172Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.08321307Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.08325858Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.102496113Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.102547314Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.12058418Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.12062566Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.138840827Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.138894717Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.157021014Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.157058874Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.173558511Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.173599171Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.191939539Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.191981469Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.210128606Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.210170186Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.230230094Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.230275284Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.249566118Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.249607818Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.267723075Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.267777105Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.284159991Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.284206252Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.301905216Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.301955946Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.320559025Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.320612595Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.320806747Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.320937277Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:32.416153357Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:32.416227808Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:32.416237668Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:32.416264208Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:32.416269608Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:32.416276168Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:32.416280688Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:32.416285058Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:32.416289348Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:32.416295248Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:32.416299798Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:32.416306068Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:32.416313698Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:32.416319858Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:32.416324418Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:32.416328618Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:32.416334888Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:32.416340638Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:32.416346578Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:32.416363818Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"3a37163b-2356-4357-b900-e9d8bc3bd204","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized)]","latency_us":473473,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:32.41659603Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"958.491049ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3a37163b-2356-4357-b900-e9d8bc3bd204\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:33:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3a37163b-2356-4357-b900-e9d8bc3bd204\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.376228691Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.376972865Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.376985885Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.3981991Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.3982533Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.420650332Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.420695772Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.439441832Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.439489973Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.458477064Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.458518615Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.477430226Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.477467066Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.496400047Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.496450687Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.516732127Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.516773947Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.53605697Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.536101961Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.555341444Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.555382224Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.575271761Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.575319101Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.595382179Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.595421199Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.614631972Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.614681462Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.633666704Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.633706474Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.651320327Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.651356627Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.669747396Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.669788866Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.689492101Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.689532852Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.707817289Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.70786886Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.728034068Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.728080638Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.745673952Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.745712962Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.769577662Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.769635423Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.769840354Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.769992975Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:33.872349576Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:33.872416717Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:33.872425387Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:33.872442567Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:33.872448397Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:33.872453817Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:33.872459137Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:33.872466007Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:33.872471527Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:33.872476797Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:33.872482427Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:33.872486197Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:33.872491487Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:33.872495367Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:33.872499497Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:33.872503297Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:33.872506877Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:33.872510508Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:33.872527957Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:33.872598378Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"e0aeaf03-4975-4752-a3bf-754a88491deb","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized)]","latency_us":496238,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:33.872820979Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"2.516912722s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e0aeaf03-4975-4752-a3bf-754a88491deb\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:33:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e0aeaf03-4975-4752-a3bf-754a88491deb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.391766627Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.392218829Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.39222792Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.415413046Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.415452246Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.436150348Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.436188938Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.457039661Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.457080581Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.475486899Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.475522439Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.494892493Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.494923503Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.513893145Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.513932215Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.533607601Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.533643841Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.552706483Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.552743153Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.572771361Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.572803631Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.592113645Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.592146455Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.611469309Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.611500609Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.630848543Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.630899953Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.649419351Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.649460811Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.669672Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.66971069Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.689305615Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.689342316Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.707744924Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.707779294Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.726002831Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.726032601Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.745433555Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.745475226Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.766449109Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.766488729Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.785701592Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.785734242Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.785931094Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.786043534Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.844181536Z","hostname":"buskin","pid":1593,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41489"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.844460998Z","hostname":"buskin","pid":1593,"uri":"/baseboard","method":"GET","req_id":"6f346d02-ddc4-40ad-b45d-404076dd62da","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41489","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":170,"response_code":"200"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:36.882737923Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:36.882798053Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:36.882805923Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:36.882814333Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:36.882821023Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:36.882827743Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:36.882834443Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:36.882841223Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:36.882847493Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:36.882853883Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:36.882861983Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:36.882868223Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:36.882876283Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:36.882882813Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:36.882901453Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:36.882909473Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:36.882915914Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:36.882923994Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:36.882930424Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:36.883019174Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"f382cc51-0951-42dc-a676-a69bec7be12b","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized)]","latency_us":491034,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:36.883219946Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.842046584s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f382cc51-0951-42dc-a676-a69bec7be12b\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:33:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f382cc51-0951-42dc-a676-a69bec7be12b\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:38.284923086Z","hostname":"buskin","pid":1593,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"2.486963091s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.72639446Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.726709372Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.726718582Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.750519722Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.750572423Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.7704979Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.77055244Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.791783135Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.791826435Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.812744148Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.812790298Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.832835036Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.832881716Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.852670163Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.852712873Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.873869457Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.873916637Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.892943499Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.89298742Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.915763194Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.915808054Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.937787153Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.937839243Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.958983268Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.959041538Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.977976299Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.97801658Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.998158168Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:38.998198298Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:39.018709199Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:39.018762049Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:39.039044508Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:39.039101649Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:39.059637499Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:39.05967871Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:39.079955259Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:39.079997819Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:39.100260008Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:39.100302599Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:39.120167725Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:39.120207945Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:39.142197005Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:39.142250075Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:39.142481046Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:39.142652067Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:39.237566025Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:39.237609566Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:39.237614816Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:39.237619146Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:39.237624496Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:39.237628566Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:39.237633696Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:39.237637986Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:39.237643096Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:39.237647286Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:39.237651276Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:39.237672826Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:39.237678166Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:39.237682346Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:39.237687486Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:39.237691596Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:39.237696666Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:39.237700846Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:39.237704206Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:39.237792437Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"f7999682-9b73-4220-9e10-17ba0f9292b2","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized)]","latency_us":511265,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:39.238020028Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"3.667256727s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f7999682-9b73-4220-9e10-17ba0f9292b2\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:33:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f7999682-9b73-4220-9e10-17ba0f9292b2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:42.906758354Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:42.907102076Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:42.907109486Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:42.931547549Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:42.931582289Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:42.954013052Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:42.954047012Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:42.976055451Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:42.976088481Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:42.996989174Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:42.997026494Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.018219119Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.018250519Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.03880144Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.0388324Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.060317476Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.060349277Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.081694782Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.081733432Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.103638411Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.103674761Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.126228934Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.126278644Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.148225203Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.148260623Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.170647715Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.170688495Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.192679235Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.192712035Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.213856989Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.213893179Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.235851978Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.235884149Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.259493067Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.259528088Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.281748768Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.281789258Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.303799868Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.303839658Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.325152813Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.325193603Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.346216717Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.346257777Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.346417268Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.346519039Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:43.451808918Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:43.451886478Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:43.451906288Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:43.451913779Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:43.451920039Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:43.451926879Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:43.451934409Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:43.451940539Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:43.451946859Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:43.451954429Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:43.451960599Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:43.451977619Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:43.451984169Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:43.451991419Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:43.451997369Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:43.452004219Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:43.452027059Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:43.452063839Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:43.4520849Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:43.45224899Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"17500081-3dfb-4f58-b1c4-18c2d2f699a7","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized)]","latency_us":545337,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:43.452474602Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"4.58136794s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"17500081-3dfb-4f58-b1c4-18c2d2f699a7\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:33:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"17500081-3dfb-4f58-b1c4-18c2d2f699a7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.035896774Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.037574584Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.037583174Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.05911391Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.061400364Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.077633049Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.07768231Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.09986198Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.09992441Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.117848355Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.117909306Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.137081759Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.137145349Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.154846603Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.154893293Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.173650234Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.173702544Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.191852721Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.191908001Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.209603515Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.209654735Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.228204234Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.228273465Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.246508932Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.246572372Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.265264552Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.265323702Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.283933292Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.283987992Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.302689692Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.302736133Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.320310686Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.320349546Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.33639634Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.336445271Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.354773648Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.354824279Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.372730364Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.372786094Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.392099298Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.392173828Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.412093435Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.412155616Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.412368127Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.412528208Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:48.505728706Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:48.505799066Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:48.505809266Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:48.505815936Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:48.505822547Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:48.505834967Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:48.505841027Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:48.505847667Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:48.505853356Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:48.505859537Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:48.505884667Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:48.505890877Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:48.505898577Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:48.505905707Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:48.505911697Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:48.505919077Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:48.505924897Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:48.505932477Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:48.505939527Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:48.506054538Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"3fc2a4c6-aafc-493e-954b-0d11b937a9a7","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized)]","latency_us":469777,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:48.506259749Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"2.545782529s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3fc2a4c6-aafc-493e-954b-0d11b937a9a7\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:33:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3fc2a4c6-aafc-493e-954b-0d11b937a9a7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.053937264Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.054468617Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.054485387Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.07365783Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.075582951Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.094756824Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.094811624Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.112617269Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.112670259Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.129705809Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.12975887Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.14848553Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.14855407Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.166351075Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.166414595Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.183912038Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.183967438Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.203253012Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.203312262Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.220419043Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.220471153Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.23874952Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.238808021Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.257134528Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.257186919Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.27442151Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.2744803Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.292695177Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.292753638Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.310396912Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.310455622Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.329593014Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.329646885Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.347721411Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.347767431Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.363316673Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.363373383Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.381412119Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.381475769Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.399508296Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.401412047Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.417191639Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.41724786Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.417446961Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.417576182Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:51.504343442Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:51.504425032Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:51.504445162Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:51.504472482Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:51.504483553Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:51.504502923Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:51.504511313Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:51.504523153Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:51.504533853Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:51.504541893Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:51.504575513Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:51.504585863Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:51.504602353Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:51.504612823Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:51.504624543Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:51.504641814Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:51.504651774Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:51.504658154Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:51.504666234Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:51.504754194Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"c0d08d35-7451-4e83-80af-a60387e84499","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized)]","latency_us":450611,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:51.504987316Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"6.61461392s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c0d08d35-7451-4e83-80af-a60387e84499\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:33:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c0d08d35-7451-4e83-80af-a60387e84499\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:55.809299818Z","hostname":"buskin","pid":1593,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"3.102571867s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.120632771Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.120995253Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.121013023Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.141913374Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.14426139Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.160349084Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.160402255Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.179480047Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.179534387Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.1987071Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.200984733Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.216415054Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.216476724Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.233611635Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.233659835Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.252535826Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.252595486Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.271418737Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.271479907Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.289231182Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.289284085Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.307775521Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.307838901Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.326739232Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.326784382Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.345699674Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.345744064Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.363768902Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.363827092Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.382765948Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.382825536Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.402167545Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.402221186Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.420537444Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.420658724Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.439502658Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.439554821Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.457643331Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.457701309Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.477065969Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.477118422Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.495957937Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.496030907Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.496225978Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.496365589Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:58.589039078Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:58.589102791Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:58.589122301Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:58.58913192Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:58.589141866Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:58.589153651Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:58.589163532Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:58.589195923Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:58.58920722Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:58.589217045Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:58.58922642Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:58.589235448Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:58.589245217Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:58.589256513Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:58.589265513Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:58.589276614Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:58.589285802Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:58.58929166Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:58.589299404Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:33:58.589373355Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"e7273ff0-c9a1-4ec4-88b0-751876065178","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized)]","latency_us":468477,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:33:58.589566453Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"3.855983128s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e7273ff0-c9a1-4ec4-88b0-751876065178\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:33:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e7273ff0-c9a1-4ec4-88b0-751876065178\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.448223603Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.448585675Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.448595215Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.470277032Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.470331493Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.491410178Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.491482665Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.510417381Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.510468842Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.529529976Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.529590398Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.548394222Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.548454109Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.566821715Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.566876859Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.584572784Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.584628664Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.603055623Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.603109153Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.621663282Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.621725073Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.639622688Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.639671358Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.658569588Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.658641689Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.677221138Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.677278148Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.696198789Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.696249479Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.715848885Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.715919605Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.734869046Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.734925227Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.753890528Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.753953289Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.772255906Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.774233798Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.791860342Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.791929867Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.81022932Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.81029073Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.829344122Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.829399722Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.829626323Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.829770984Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:02.923243834Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:02.923336184Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:02.923351054Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:02.923361114Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:02.923379805Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:02.923391905Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:02.923407795Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:02.923416025Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:02.923422285Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:02.923428415Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:02.923437645Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:02.923447065Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:02.923456455Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:02.923471305Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:02.923486555Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:02.923494135Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:02.923500735Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:02.923506115Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:02.923513585Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:02.923521425Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"88e6d25e-dc1a-4b92-9e37-3fafdf727e1f","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized)]","latency_us":475024,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:02.923741167Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"3.457766996s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"88e6d25e-dc1a-4b92-9e37-3fafdf727e1f\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:34:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"88e6d25e-dc1a-4b92-9e37-3fafdf727e1f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.382657279Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.383754831Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.383764771Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.407236589Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.407291929Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.426727513Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.426798793Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.445336193Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.445410313Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.462928616Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.463004816Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.48002002Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.480073439Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.498981602Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.499047434Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.516963914Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.517016524Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.536222198Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.536275092Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.553919011Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.553982371Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.572172278Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.572221158Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.590197114Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.590247734Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.608393663Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.60845056Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.627023469Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.627115991Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.645298767Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.645348118Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.664123994Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.664189544Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.682066473Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.682123154Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.700041699Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.700106969Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.717267785Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.719543943Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.734998014Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.735051585Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.753242892Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.753318322Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.753545283Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.753712304Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:06.843720333Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:06.843793584Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:06.843803544Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:06.843812474Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:06.843820874Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:06.843827674Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:06.843857284Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:06.843869044Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:06.843879124Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:06.843891914Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:06.843902124Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:06.843911934Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:06.843923774Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:06.843931805Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:06.843941574Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:06.843951675Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:06.843961265Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:06.843970825Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:06.843976785Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.844078025Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"aa8e1afc-9a0a-455b-aa92-096901529541","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized)]","latency_us":461077,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:06.844326387Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"7.017581671s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"aa8e1afc-9a0a-455b-aa92-096901529541\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:34:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"aa8e1afc-9a0a-455b-aa92-096901529541\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.879374123Z","hostname":"buskin","pid":1593,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44188"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:06.879477023Z","hostname":"buskin","pid":1593,"uri":"/baseboard","method":"GET","req_id":"bcd0c2a2-793e-4d2f-b448-393ae301f44c","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44188","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":52,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:13.864635759Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:13.865020015Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:13.865030045Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:13.88792204Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:13.88798947Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:13.908855407Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:13.908915453Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:13.930826507Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:13.930878792Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:13.950574796Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:13.950628243Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:13.95768423Z","hostname":"buskin","pid":1593,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"7.432261137s"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:13.971732393Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:13.971799193Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:13.992543455Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:13.992599995Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.013880864Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.01393042Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.035203635Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.035263576Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.05806839Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.05812116Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.079553086Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.079591786Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.100492947Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.100559575Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.121575063Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.121630323Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.141676161Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.141735672Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.162477803Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.162536044Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.182812193Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.182869955Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.204724702Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.204787242Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.225078151Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.225139932Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.243716026Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.243769633Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.265103587Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.265156557Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.284819553Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.284889784Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.285080229Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.285201786Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:14.387481096Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:14.387575097Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:14.387584357Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:14.387593327Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:14.387600997Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:14.387607357Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:14.387615027Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:14.387631907Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:14.387639907Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:14.387647387Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:14.387653367Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:14.387665377Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:14.387671477Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:14.387679337Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:14.387686937Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:14.387710397Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:14.387717687Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:14.387723517Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:14.387731147Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:14.387835058Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"b4a7b511-621b-402b-b235-9fc2d0a6333f","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized)]","latency_us":522856,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:14.388069049Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"12.894142905s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b4a7b511-621b-402b-b235-9fc2d0a6333f\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:34:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b4a7b511-621b-402b-b235-9fc2d0a6333f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.286999312Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.287815697Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.287830847Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.311227249Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.311358425Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.331896306Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.331964586Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.351482691Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.351545791Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.369650568Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.369723388Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.386727508Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.386796639Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.404955114Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.405012621Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.423958121Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.424016341Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.442072114Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.442124594Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.461623742Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.461678239Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.480845561Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.480900662Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.498686863Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.498762783Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.517569776Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.517626346Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.535165711Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.535225061Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.551990175Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.552049218Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.570245797Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.570307737Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.587963701Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.588025681Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.605539884Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.607406303Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.62260314Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.622657647Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.640432229Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.64049171Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.658465945Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.658541995Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.658859407Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.659029798Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:27.751198018Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:27.751287905Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:27.751298649Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:27.751308756Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:27.751316452Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:27.751325846Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:27.751343219Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:27.75135295Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:27.751362362Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:27.751373575Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:27.751457501Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:27.751481711Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:27.751490561Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:27.751496651Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:27.751511501Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:27.751530491Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:27.751538792Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:27.751544721Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:27.751560772Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:27.751586872Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"d2a35c31-9d39-4f5a-9c4f-6e03f067c475","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized)]","latency_us":463980,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:27.751814083Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"9.91621435s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d2a35c31-9d39-4f5a-9c4f-6e03f067c475\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:34:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d2a35c31-9d39-4f5a-9c4f-6e03f067c475\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:36.433473393Z","hostname":"buskin","pid":1593,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"16.079319507s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:36.920611838Z","hostname":"buskin","pid":1593,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33305"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:36.920686661Z","hostname":"buskin","pid":1593,"uri":"/baseboard","method":"GET","req_id":"7ed847fa-7aea-4360-a92f-8266677ed6cb","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33305","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.671381867Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.671679769Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.671692341Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.692657542Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.692717002Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.712873011Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.715203314Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.732125163Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.732177634Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.749520626Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.749569956Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.767762363Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.767817614Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.785202536Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.785263716Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.80301799Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.803071031Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.821008946Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.821076027Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.839742346Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.839811097Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.857760752Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.857813943Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.876713034Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.876772864Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.895049241Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.895108412Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.9123311Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.912388334Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.930751001Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.930819332Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.950062134Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.950119485Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.967453577Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.967512237Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.985130551Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:37.985188001Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:38.00283911Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:38.002880425Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:38.021537929Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:38.021580388Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:38.039894162Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:38.039945963Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:38.040203869Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:38.040342311Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:38.133410462Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:38.133491113Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:38.133501933Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:38.133510693Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:38.133518633Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:38.133526303Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:38.133534703Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:38.133544923Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:38.133575193Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:38.133584963Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:38.133594423Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:38.133611073Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:38.133621064Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:38.133631344Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:38.133641353Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:38.133651324Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:38.133661174Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:38.133670774Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:38.133676644Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:38.133774404Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"0bb8d76d-8b6d-493d-b14d-eca623f9b2a4","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized)]","latency_us":462119,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:38.134002396Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"10.3525632s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0bb8d76d-8b6d-493d-b14d-eca623f9b2a4\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:34:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0bb8d76d-8b6d-493d-b14d-eca623f9b2a4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.49079044Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.491138254Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.491147722Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.513451593Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.515980178Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.534816509Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.534873209Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.554075232Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.554125802Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.571610475Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.571681415Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.589573665Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.589632015Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.607922928Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.607978639Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.626460985Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.626517713Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.644021181Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.644072251Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.661784065Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.663664556Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.680959028Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.681012228Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.700404182Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.700471332Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.718512498Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.718579159Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.736690385Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.736759726Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.754462782Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.754511551Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.772223914Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.772275674Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.79027452Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.790331941Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.808113545Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.808162045Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.827201137Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.827262558Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.845178473Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.847452326Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.864173834Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.864261775Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.864492916Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.864619307Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:48.95699809Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:48.95706714Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:48.957076511Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:48.957099166Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:48.95710616Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:48.957113866Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:48.957120251Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:48.957128332Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:48.957145413Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:48.957153223Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:48.957161407Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:48.957169713Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:48.957176135Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:48.957190376Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:48.957196479Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:48.95720472Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:48.957219261Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:48.957225655Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:48.957230782Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:34:48.957315589Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"ecc80f57-8490-47db-ae40-086e4f7f6d18","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized)]","latency_us":466259,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:34:48.957528469Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"18.949901779s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ecc80f57-8490-47db-ae40-086e4f7f6d18\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:34:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ecc80f57-8490-47db-ae40-086e4f7f6d18\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:06.962055474Z","hostname":"buskin","pid":1593,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54615"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:06.962124164Z","hostname":"buskin","pid":1593,"uri":"/baseboard","method":"GET","req_id":"3918c266-3822-41db-a03b-2a03103cf991","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54615","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:07.555875163Z","hostname":"buskin","pid":1593,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"40.247931112s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:07.913223274Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:07.913534825Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:07.913543113Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:07.935490625Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:07.935567345Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:07.956061786Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:07.956131336Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:07.975254059Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:07.975312639Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:07.994372461Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:07.994429661Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.012739279Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.012805079Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.033111679Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.035443452Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.050751742Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.052679154Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.069635553Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.069695354Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.087337197Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.087387787Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.10652734Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.106572283Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.125024129Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.125085979Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.1422716Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.142330621Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.160873379Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.16093963Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.178918296Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.178983926Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.19664947Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.19670817Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.214759575Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.21481501Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.233416736Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.233479776Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.251928565Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.251997945Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.26975807Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.26981866Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.285670373Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.285732353Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.286014415Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.286162116Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:08.375119499Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:08.375186059Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:08.375211959Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:08.375218009Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:08.375226809Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:08.37523563Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:08.37524273Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:08.375251179Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:08.375265Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:08.37527525Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:08.37528207Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:08.37529718Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:08.37531237Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:08.37532096Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:08.37532741Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:08.37533591Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:08.37534476Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:08.3753582Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:08.37536623Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:08.375466791Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"46ff6853-869f-4429-bd24-2b89e5e99288","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized)]","latency_us":462032,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:08.375707012Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"27.278658679s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"46ff6853-869f-4429-bd24-2b89e5e99288\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:35:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"46ff6853-869f-4429-bd24-2b89e5e99288\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.660950093Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.661230625Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.661251765Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.682852391Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.682920431Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.701271059Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.703528992Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.720822114Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.720918095Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.741305914Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.741355405Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.75917271Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.75922123Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.776658332Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.776727943Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.795251392Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.795304542Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.812856635Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.812923179Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.831195983Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.831259393Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.849727262Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.849790732Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.868273571Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.868323331Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.88632582Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.886449538Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.904108151Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.904155922Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.92255749Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.92261719Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.940780717Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.940844747Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.958650432Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.958711992Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.978026536Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.978089576Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.996602735Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:35.996667715Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:36.015273245Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:36.015321085Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:36.03322328Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:36.033279841Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:36.033564557Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:36.033724008Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:36.127750706Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:36.127847397Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:36.127855746Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:36.127863227Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:36.127871087Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:36.127878317Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:36.127883947Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:36.127892967Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:36.127900767Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:36.127908297Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:36.127914357Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:36.127923597Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:36.127928747Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:36.127937527Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:36.127944647Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:36.127953517Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:36.127972817Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:36.127982567Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:36.127996598Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:36.128148368Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"26cf176d-2d88-462c-ae8f-5c8d8d132913","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\", TimeNotSynchronized), (\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\", TimeNotSynchronized), (\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\", TimeNotSynchronized), (\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\", TimeNotSynchronized), (\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\", TimeNotSynchronized), (\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\", TimeNotSynchronized), (\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\", TimeNotSynchronized), (\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\", TimeNotSynchronized), (\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\", TimeNotSynchronized), (\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\", TimeNotSynchronized), (\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\", TimeNotSynchronized), (\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\", TimeNotSynchronized), (\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\", TimeNotSynchronized), (\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\", TimeNotSynchronized), (\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\", TimeNotSynchronized), (\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\", TimeNotSynchronized), (\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\", TimeNotSynchronized), (\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\", TimeNotSynchronized), (\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\", TimeNotSynchronized)]","latency_us":467009,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:35:36.12840219Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"35.55166526s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"26cf176d-2d88-462c-ae8f-5c8d8d132913\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:35:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"26cf176d-2d88-462c-ae8f-5c8d8d132913\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:36.998838575Z","hostname":"buskin","pid":1593,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53597"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:35:36.998909609Z","hostname":"buskin","pid":1593,"uri":"/baseboard","method":"GET","req_id":"86c9c379-3ea6-489a-b162-6c5c086c9678","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53597","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":41,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:02.847887086Z","hostname":"buskin","pid":1593,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"88.483029223s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:07.036384695Z","hostname":"buskin","pid":1593,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56922"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:07.036426345Z","hostname":"buskin","pid":1593,"uri":"/baseboard","method":"GET","req_id":"1c97d88f-a4cc-45af-8161-257445521d2b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56922","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":29,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dd2b8493-b48e-4834-ad8e-a53d21815a61, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cc505437-f47e-4e85-84e3-c10ec252eef7, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6151b150-2c83-46b9-8dd9-d454da5235bf, kind: Service { id: 6d4f2f0d-0048-43a0-b797-4120ab6bb8ed }, name: Name(\"ntp-6d4f2f0d-0048-43a0-b797-4120ab6bb8ed\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 79])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 45e11811-5e91-4520-b80a-7f642e885b6d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: f44a281c-ca8a-47a1-9feb-a5b347543709, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: ec283d93-6725-4976-bed2-c328ce4398e5, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99ae6502-7f59-4787-8a1e-0923522db0a2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cfa9067d-2711-42a0-944b-e859db1c0817, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63642991-e779-4349-8697-9bfc6b96d7e4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f18a2ae7-5a47-47b8-9eb2-b47a19a2f818, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 44fe13c7-2d3b-4321-9d0e-0f378ee54327, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 109e60dc-df19-44bb-8891-e91ae9e42944, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External } } } }, OmicronZoneConfig { id: d0b8200e-a07a-4794-b87b-17a74dc261e4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External } } } }, OmicronZoneConfig { id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External } } } }, OmicronZoneConfig { id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External } } } }, OmicronZoneConfig { id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External } } } }, OmicronZoneConfig { id: 46326447-e170-4123-8268-47a9a79585dd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External } } } }, OmicronZoneConfig { id: 10edf729-c77a-4f8c-9d40-496c39338dcb, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8f204fd-3d76-4652-963a-9ee349d4a396, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.686259854Z","hostname":"buskin","pid":1593,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.68673328Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.686741277Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd2b8493-b48e-4834-ad8e-a53d21815a61, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.707603659Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.707653685Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cc505437-f47e-4e85-84e3-c10ec252eef7, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.727012503Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.727069194Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a2adbd8-5cd9-40a0-9c4f-33c9b2f43703, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.745592919Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.745635331Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 45e11811-5e91-4520-b80a-7f642e885b6d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.765228918Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.765281268Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae06d4ce-795c-4d42-b79d-9db3f61d73a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.784478141Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.784533801Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14c4878d-c5dd-4fc2-8846-aaf3152f4120, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.800851397Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.800904327Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f44a281c-ca8a-47a1-9feb-a5b347543709, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.818465731Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.818517501Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8a2bf75b-8c7f-4d8d-9ede-6fc064124d25, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.835790933Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.835847723Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8b17fb63-5764-4496-bed6-acd4016c4f2d, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.853203045Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.853265859Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ec283d93-6725-4976-bed2-c328ce4398e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.871897185Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.871947365Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c94f186-c252-4046-8d01-e8c5cf6a3620, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.88971512Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.8897686Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 109e60dc-df19-44bb-8891-e91ae9e42944, dataset_name: DatasetName { pool_name: ZpoolName { id: 58eb44a6-f8c0-46ec-bcea-52bf3efa8996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.907401904Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.907450704Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d0b8200e-a07a-4794-b87b-17a74dc261e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 888a035b-7ecc-4eff-a005-8001c3d3ccff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.924761646Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.924814296Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fe5d8209-53c4-40d5-afba-39343d7ed6c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1afb2a46-5b47-46d7-9926-14b7e21c6407 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.942767301Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.942824792Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63dcd254-bbd3-489b-9df5-0db1a21b2d37, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c0211ed-cba6-48d8-8b63-a5f21383d980 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.960920968Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.960967838Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39fc7bb0-36ee-45b1-96e7-097a2d543d2a, dataset_name: DatasetName { pool_name: ZpoolName { id: fc47eb1d-9a62-49dd-845d-82b29d12e969 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.978361131Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.978410101Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 46326447-e170-4123-8268-47a9a79585dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a3be27-a8c4-4992-8657-cb4aacfdb8ab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.996192466Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:11.996249576Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10edf729-c77a-4f8c-9d40-496c39338dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: 9edc3ed7-0367-4fc4-87ff-888719d0b682 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.016985398Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.017046358Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8f204fd-3d76-4652-963a-9ee349d4a396, dataset_name: DatasetName { pool_name: ZpoolName { id: ed53811e-9a91-4270-bd15-bcb96ef98de1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.036692524Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.036747484Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f05d09e0-db61-4cdb-9dbd-1b8480c392bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.053621763Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.053690413Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.053968055Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.054115826Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.138868214Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.151977101Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.152222243Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.164736356Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.16539994Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.176082903Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.218958165Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.219024645Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.219043015Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.219050125Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.219056765Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.365715367Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.365967359Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.365990839Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.366013819Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.400185825Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.400230188Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.400235767Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.40024022Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.400881614Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.407151541Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.412911115Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.418113595Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.423193795Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.427576991Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_46326447-e170-4123-8268-47a9a79585dd","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.432007246Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.568284468Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.573648049Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.579337133Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.584074791Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.588686398Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.593104514Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_46326447-e170-4123-8268-47a9a79585dd","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.597864965Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.602824746Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.607902961Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.613077221Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.618621894Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.624080456Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.629688436Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.635346792Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.642040211Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.647722805Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.653123287Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.659614465Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.665214538Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.671317664Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.677305219Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.736505696Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.809408925Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.875641484Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.935425405Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.997276529Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.998523636Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:12.998538226Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:13.173994468Z","hostname":"buskin","pid":1593,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 187, 228]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte1\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:13.174096838Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:13.178680065Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:13.178694735Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:13.317813733Z","hostname":"buskin","pid":1593,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 155, 115]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte2\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:13.317896803Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:13.321044422Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:13.321068592Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:13.436077298Z","hostname":"buskin","pid":1593,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 188, 158]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte3\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:13.436156138Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:13.438834154Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:13.438859694Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:13.561574715Z","hostname":"buskin","pid":1593,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 160, 210]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte4\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:13.561661596Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:13.566773606Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:13.566809276Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:14.170597274Z","hostname":"buskin","pid":1593,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 170, 42]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte5\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:14.170647645Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:14.170673415Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:14.170681845Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:14.170694715Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:14.170702925Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:14.170712825Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:14.170720525Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:14.170730045Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:14.170737085Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:14.766735318Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:14.848163127Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:14.858995811Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:14.859019981Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:14.859152032Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:14.859545124Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:14.859574354Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:14.864456783Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:14.868968329Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:14.868989139Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:14.982925389Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:14.995448623Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:15.170390271Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:15.298631055Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:15.805412383Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:15.812735186Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:15.820966434Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:15.834383113Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:15.898055187Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:15.908515399Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:15.91705141Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:15.94448783Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:28.739368309Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:28.750540735Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:28.850354082Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:28.97947195Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:29.126931597Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:29.127750182Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:29.127962963Z","hostname":"buskin","pid":1593,"zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:29.133956239Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:29.177590025Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:29.177927867Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:29.310882909Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:29.31115974Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:29.412209885Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:29.412451986Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:29.428200479Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:29.428704922Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:29.436186215Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:29.436363236Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:29.562582569Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:29.562924091Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:29.635588738Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:29.637283378Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:29.637415538Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:29.643320273Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\") failed: Property not found. retry in 26.240249ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.73845845Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\") failed: Property not found. retry in 53.178892ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.781309433Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\") failed: Property not found. retry in 43.116266ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.791006799Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\") failed: Property not found. retry in 68.162002ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.800043053Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\") failed: Property not found. retry in 39.732598ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.809223717Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\") failed: Property not found. retry in 28.553733ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.818096589Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\") failed: Property not found. retry in 72.727406ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.825858284Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\") failed: Property not found. retry in 45.708047ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.834908917Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\") failed: Property not found. retry in 26.595939ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.842680863Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\") failed: Property not found. retry in 70.632796ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.850199487Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\") failed: Property not found. retry in 51.286465ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.857067848Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\") failed: Property not found. retry in 65.580436ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.86420529Z","hostname":"buskin","pid":1593,"zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\") failed: Property not found. retry in 52.053555ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.872256327Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\") failed: Property not found. retry in 140.875091ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.881535952Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\") failed: Property not found. retry in 98.386162ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.890188222Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\") failed: Property not found. retry in 51.058399ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.897461685Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\") failed: Property not found. retry in 66.070276ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.905491263Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\") failed: Property not found. retry in 139.906214ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.912947546Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\") failed: Property not found. retry in 114.165517ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.919972668Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\") failed: Property not found. retry in 88.564371ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.931247274Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\") failed: Property not found. retry in 120.780191ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.940325317Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\") failed: Property not found. retry in 271.261605ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.947736991Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\") failed: Property not found. retry in 103.662769ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.955382516Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\") failed: Property not found. retry in 126.685867ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.964830141Z","hostname":"buskin","pid":1593,"zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\") failed: Property not found. retry in 133.825562ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.975248463Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\") failed: Property not found. retry in 236.674053ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.982450915Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\") failed: Property not found. retry in 180.551232ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:30.992834686Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\") failed: Property not found. retry in 240.221336ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:31.001015054Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\") failed: Property not found. retry in 296.129404ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:31.031725915Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\") failed: Property not found. retry in 268.010098ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:31.042325477Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\") failed: Property not found. retry in 191.80316ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:31.05302412Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\") failed: Property not found. retry in 170.690819ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:31.067288014Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\") failed: Property not found. retry in 193.807224ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:31.075336821Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\") failed: Property not found. retry in 212.611633ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:31.083774761Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\") failed: Property not found. retry in 297.823322ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:31.10065668Z","hostname":"buskin","pid":1593,"zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\") failed: Property not found. retry in 295.772214ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:31.119598671Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:31.141243789Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:31.14152138Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\") failed: Property not found. retry in 514.668904ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:31.182925334Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\") failed: Property not found. retry in 391.023869ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:31.230894905Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\") failed: Property not found. retry in 308.580609ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:31.240684993Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\") failed: Property not found. retry in 363.840751ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:31.251057384Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\") failed: Property not found. retry in 565.138149ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:31.262004528Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\") failed: Property not found. retry in 353.652566ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:31.271046392Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\") failed: Property not found. retry in 512.251693ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:31.280483697Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\") failed: Property not found. retry in 385.347871ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:31.30810925Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\") failed: Property not found. retry in 524.311605ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:31.3217554Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\") failed: Property not found. retry in 56.197743ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:31.329440185Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\") failed: Property not found. retry in 385.50345ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:31.39319994Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\") failed: Property not found. retry in 70.675487ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.048823553Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\") failed: Property not found. retry in 432.461176ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.087229599Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\") failed: Property not found. retry in 748.945591ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.112312126Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\") failed: Property not found. retry in 525.715027ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.15042309Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\") failed: Property not found. retry in 1.09426451s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.182170307Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\") failed: Property not found. retry in 759.422099ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.213639172Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_46326447-e170-4123-8268-47a9a79585dd:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:32.217696506Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\") failed: Property not found. retry in 1.014910202s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.249113431Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\") failed: Property not found. retry in 593.508026ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.27455063Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\") failed: Property not found. retry in 366.322092ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.301866301Z","hostname":"buskin","pid":1593,"zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\") failed: Property not found. retry in 436.759509ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.324795156Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\") failed: Property not found. retry in 1.116294607s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.355503196Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\") failed: Property not found. retry in 536.102844ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.378444191Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\") failed: Property not found. retry in 460.781532ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.401278195Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\") failed: Property not found. retry in 278.794576ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.421696595Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:32.421863686Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:32.508507316Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:32.508605486Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\") failed: Property not found. retry in 674.081642ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.533579333Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\") failed: Property not found. retry in 57.62696ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.673667306Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\") failed: Property not found. retry in 760.426357ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.687647279Z","hostname":"buskin","pid":1593,"zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\") failed: Property not found. retry in 1.202995759s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.702934629Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\") failed: Property not found. retry in 418.369256ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.717276653Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\") failed: Property not found. retry in 132.533239ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.74578337Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\") failed: Property not found. retry in 862.612522ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.77803923Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\") failed: Property not found. retry in 71.709752ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.790413273Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\") failed: Property not found. retry in 1.409299529s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.873930954Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\") failed: Property not found. retry in 1.157666787s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.887508284Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\") failed: Property not found. retry in 202.391966ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.901567286Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\") failed: Property not found. retry in 1.246317588s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.915126266Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\") failed: Property not found. retry in 51.740622ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.925330006Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\") failed: Property not found. retry in 805.707717ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.938306222Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\") failed: Property not found. retry in 1.350168274s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:32.989090601Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\") failed: Property not found. retry in 209.431341ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:33.004696783Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\") failed: Property not found. retry in 584.422222ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:33.118480061Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\") failed: Property not found. retry in 401.838851ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:33.149258033Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\") failed: Property not found. retry in 1.017835728s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:33.222375032Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\") failed: Property not found. retry in 432.558266ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:33.234664355Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\") failed: Property not found. retry in 592.473551ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:33.278241641Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\") failed: Property not found. retry in 935.46307ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:33.290351952Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\") failed: Property not found. retry in 1.316757249s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:33.459903019Z","hostname":"buskin","pid":1593,"zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\") failed: Property not found. retry in 1.331799911s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:33.481928018Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\") failed: Property not found. retry in 1.004186832s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:33.561925039Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\") failed: Property not found. retry in 1.187379892s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:33.650939932Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\") failed: Property not found. retry in 985.258947ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:33.677706339Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\") failed: Property not found. retry in 869.649287ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:33.71373591Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\") failed: Property not found. retry in 1.390138815s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:33.753844996Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:33.807966944Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:33.808024534Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\") failed: Property not found. retry in 870.38097ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:33.881081294Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\") failed: Property not found. retry in 611.967779ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:33.917345607Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\") failed: Property not found. retry in 63.697335ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:33.974387692Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\") failed: Property not found. retry in 81.861876ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:34.047685803Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\") failed: Property not found. retry in 981.835161ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:34.05560192Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\") failed: Property not found. retry in 219.480298ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:34.142234489Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\") failed: Property not found. retry in 1.220302097s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:34.170236414Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\") failed: Property not found. retry in 858.951561ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:34.235349517Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\") failed: Property not found. retry in 563.115351ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:34.250535366Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\") failed: Property not found. retry in 591.5624ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:34.292869965Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\") failed: Property not found. retry in 991.523416ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:34.348656883Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\") failed: Property not found. retry in 421.294221ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:34.369707696Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\") failed: Property not found. retry in 1.454022306s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:34.538239617Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\") failed: Property not found. retry in 945.183939ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:34.576541692Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\") failed: Property not found. retry in 922.584323ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:34.592672347Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\") failed: Property not found. retry in 1.34250957s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:34.672490236Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\") failed: Property not found. retry in 522.450743ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:34.760650064Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\") failed: Property not found. retry in 836.655449ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:34.78556342Z","hostname":"buskin","pid":1593,"zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\") failed: Property not found. retry in 677.400417ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:34.801276303Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\") failed: Property not found. retry in 786.535491ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:34.821979664Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\") failed: Property not found. retry in 911.250188ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:34.828665714Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\") failed: Property not found. retry in 819.533945ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:34.847619455Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\") failed: Property not found. retry in 1.486940988s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:34.89447626Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\") failed: Property not found. retry in 1.02363121s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:35.047221018Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\") failed: Property not found. retry in 932.708439ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:35.103504219Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\") failed: Property not found. retry in 977.928125ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:35.154068677Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\") failed: Property not found. retry in 1.278813496s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:35.293872188Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\") failed: Property not found. retry in 1.035899401s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:35.35021388Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\") failed: Property not found. retry in 1.090723902s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:35.398998606Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\") failed: Property not found. retry in 932.627272ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:35.487884639Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\") failed: Property not found. retry in 1.407729447s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:35.523673559Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\") failed: Property not found. retry in 564.016232ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:35.531963308Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\") failed: Property not found. retry in 1.32885421s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:35.61726474Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\") failed: Property not found. retry in 1.085305425s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:35.631467233Z","hostname":"buskin","pid":1593,"zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\") failed: Property not found. retry in 774.25541ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:35.675978305Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\") failed: Property not found. retry in 870.337598ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:35.747879637Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\") failed: Property not found. retry in 1.442521455s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:36.002789835Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\") failed: Property not found. retry in 697.115127ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:36.025318468Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\") failed: Property not found. retry in 1.32435415s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:36.046666773Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\") failed: Property not found. retry in 1.091314369s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:36.081442908Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\") failed: Property not found. retry in 1.005245341s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:36.105294908Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\") failed: Property not found. retry in 998.760893ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:36.142031274Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\") failed: Property not found. retry in 1.072506703s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:36.392444716Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\") failed: Property not found. retry in 600.045071ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:36.401230637Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\") failed: Property not found. retry in 1.059514746s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:36.432558842Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\") failed: Property not found. retry in 1.393589087s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:36.461742363Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\") failed: Property not found. retry in 569.956572ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:36.499576456Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\") failed: Property not found. retry in 583.808565ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:36.583152427Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\") failed: Property not found. retry in 760.41278ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:36.629427039Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\") failed: Property not found. retry in 1.136559917s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:36.727149963Z","hostname":"buskin","pid":1593,"zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\") failed: Property not found. retry in 1.372941608s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:36.736651529Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\") failed: Property not found. retry in 1.360479473s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:36.940733048Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\") failed: Property not found. retry in 1.256910208s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:36.986665358Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\") failed: Property not found. retry in 1.175175661s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:37.016688505Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\") failed: Property not found. retry in 1.130376017s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:37.103777557Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\") failed: Property not found. retry in 889.515424ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:37.124273813Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\") failed: Property not found. retry in 884.510187ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:37.192611369Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\") failed: Property not found. retry in 787.979023ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:37.20635858Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\") failed: Property not found. retry in 841.40671ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:37.218592942Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\") failed: Property not found. retry in 672.672599ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:37.382586226Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\") failed: Property not found. retry in 965.581038ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:37.399764987Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:37.405946023Z","hostname":"buskin","pid":1593,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46549"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:37.429910784Z","hostname":"buskin","pid":1593,"uri":"/baseboard","method":"GET","req_id":"8546b114-bb36-4e6b-a212-3ff28b886101","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46549","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1861,"response_code":"200"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\") failed: Property not found. retry in 1.220783509s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:37.455293853Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\") failed: Property not found. retry in 806.754208ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:37.475663533Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\") failed: Property not found. retry in 1.168678304s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:37.501137573Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:37.789276626Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:37.793668712Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\") failed: Property not found. retry in 990.629941ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:37.865414144Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\") failed: Property not found. retry in 1.250761585s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:37.873488161Z","hostname":"buskin","pid":1593,"zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\") failed: Property not found. retry in 38.880335ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:37.972051221Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\") failed: Property not found. retry in 1.061361909s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.020394985Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\") failed: Property not found. retry in 1.003308593s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.028727934Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\") failed: Property not found. retry in 137.129711ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.036212058Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\") failed: Property not found. retry in 511.666489ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.075578169Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\") failed: Property not found. retry in 930.383148ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.083662557Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\") failed: Property not found. retry in 1.140352923s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.108811234Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:38.109059196Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:38.109185607Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\") failed: Property not found. retry in 976.918499ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.122061512Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\") failed: Property not found. retry in 295.611396ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.182993441Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\") failed: Property not found. retry in 966.245255ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.200723315Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\") failed: Property not found. retry in 781.242864ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.243103934Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\") failed: Property not found. retry in 1.160434586s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.252127457Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\") failed: Property not found. retry in 36.989995ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.270225003Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\") failed: Property not found. retry in 1.465646513s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.291031216Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\") failed: Property not found. retry in 576.330665ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.31048473Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\") failed: Property not found. retry in 85.21141ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.317588562Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\") failed: Property not found. retry in 984.670932ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.37511888Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\") failed: Property not found. retry in 186.133398ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.432469117Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\") failed: Property not found. retry in 379.086628ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.489739124Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\") failed: Property not found. retry in 696.943741ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.597228416Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\") failed: Property not found. retry in 311.478462ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.628344779Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\") failed: Property not found. retry in 906.663329ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.679267148Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\") failed: Property not found. retry in 1.096136084s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.688263981Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\") failed: Property not found. retry in 953.047876ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.866492579Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\") failed: Property not found. retry in 538.530553ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.878725721Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\") failed: Property not found. retry in 818.360613ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.896600126Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\") failed: Property not found. retry in 623.688366ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:38.962344892Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\") failed: Property not found. retry in 709.340115ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:39.028877703Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\") failed: Property not found. retry in 1.283749936s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:39.045117389Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\") failed: Property not found. retry in 1.112646585s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:39.062583352Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\") failed: Property not found. retry in 1.174342195s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:39.096952367Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\") failed: Property not found. retry in 1.207282355s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:39.115217741Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\") failed: Property not found. retry in 854.280268ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:39.151756636Z","hostname":"buskin","pid":1593,"zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\") failed: Property not found. retry in 761.403829ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:39.190818155Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\") failed: Property not found. retry in 1.435938587s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:39.260789507Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\") failed: Property not found. retry in 731.722939ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:39.303368367Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\") failed: Property not found. retry in 1.134867901s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:39.368585871Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\") failed: Property not found. retry in 1.143028717s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:39.422549108Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\") failed: Property not found. retry in 667.300137ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:39.431186159Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\") failed: Property not found. retry in 714.431991ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:39.614317385Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\") failed: Property not found. retry in 641.147554ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:39.623886171Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\") failed: Property not found. retry in 946.207023ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:39.72589858Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\") failed: Property not found. retry in 739.027812ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:39.747416787Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\") failed: Property not found. retry in 1.364372887s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:39.765942096Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\") failed: Property not found. retry in 1.004372271s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:39.79390026Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\") failed: Property not found. retry in 1.26253919s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:39.828850765Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\") failed: Property not found. retry in 597.404688ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:39.961297754Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\") failed: Property not found. retry in 866.436004ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:40.015563613Z","hostname":"buskin","pid":1593,"zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\") failed: Property not found. retry in 529.465938ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:40.044950686Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\") failed: Property not found. retry in 1.147181515s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:40.107677985Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\") failed: Property not found. retry in 1.079109044s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:40.186669879Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\") failed: Property not found. retry in 1.416402814s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:40.278409388Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\") failed: Property not found. retry in 817.233016ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:40.289110201Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\") failed: Property not found. retry in 681.377766ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:40.332468346Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\") failed: Property not found. retry in 1.454084917s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:40.341928212Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\") failed: Property not found. retry in 1.0025561s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:40.350728073Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\") failed: Property not found. retry in 1.006069844s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:40.497287495Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\") failed: Property not found. retry in 1.429909751s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:40.513931383Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\") failed: Property not found. retry in 1.160791991s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:40.569987153Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\") failed: Property not found. retry in 503.536819ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:40.579950251Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\") failed: Property not found. retry in 1.019851772s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:40.588495251Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\") failed: Property not found. retry in 503.084836ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:40.685853083Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\") failed: Property not found. retry in 1.383055051s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:40.709888054Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\") failed: Property not found. retry in 871.966096ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:40.810653636Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\") failed: Property not found. retry in 728.53225ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:40.894514969Z","hostname":"buskin","pid":1593,"zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\") failed: Property not found. retry in 947.704321ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.029697754Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\") failed: Property not found. retry in 554.887447ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.101496956Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\") failed: Property not found. retry in 903.978767ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.115600079Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\") failed: Property not found. retry in 872.459744ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.128126553Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\") failed: Property not found. retry in 566.138551ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.158621202Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:41.164285006Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:41.168714641Z","hostname":"buskin","pid":1593,"zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\") failed: Property not found. retry in 929.832504ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.203561756Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\") failed: Property not found. retry in 695.757749ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.26876256Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\") failed: Property not found. retry in 524.05309ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.28237924Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\") failed: Property not found. retry in 1.393926623s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.367729411Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\") failed: Property not found. retry in 26.300056ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.434738765Z","hostname":"buskin","pid":1593,"zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\") failed: Property not found. retry in 81.901266ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.475177333Z","hostname":"buskin","pid":1593,"zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\") failed: Property not found. retry in 1.260597851s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.516356845Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\") failed: Property not found. retry in 260.22775ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.582933867Z","hostname":"buskin","pid":1593,"zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\") failed: Property not found. retry in 874.678344ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.623483435Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\") failed: Property not found. retry in 909.126333ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.635419945Z","hostname":"buskin","pid":1593,"zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\") failed: Property not found. retry in 1.073554314s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.669473454Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\") failed: Property not found. retry in 1.389477876s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.696330023Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\") failed: Property not found. retry in 914.414902ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.70937947Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\") failed: Property not found. retry in 1.013798165s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.737582446Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\") failed: Property not found. retry in 502.904241ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.749904108Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\") failed: Property not found. retry in 931.396711ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.816795991Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\") failed: Property not found. retry in 522.203326ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.830600352Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\") failed: Property not found. retry in 498.473362ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.858263685Z","hostname":"buskin","pid":1593,"zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\") failed: Property not found. retry in 859.93558ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.955226065Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\") failed: Property not found. retry in 808.824768ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.975465614Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\") failed: Property not found. retry in 1.292153562s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:41.990737334Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\") failed: Property not found. retry in 878.292797ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:42.014882336Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\") failed: Property not found. retry in 865.139125ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:42.033311464Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\") failed: Property not found. retry in 818.942349ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:42.105051836Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\") failed: Property not found. retry in 798.109296ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:42.152775026Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\") failed: Property not found. retry in 778.127007ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:42.266515765Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\") failed: Property not found. retry in 548.119653ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:42.37116387Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\") failed: Property not found. retry in 633.03023ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:42.385194893Z","hostname":"buskin","pid":1593,"zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\") failed: Property not found. retry in 601.26314ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:42.512477131Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\") failed: Property not found. retry in 854.738253ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:42.562213244Z","hostname":"buskin","pid":1593,"zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\") failed: Property not found. retry in 727.340194ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:42.646825621Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\") failed: Property not found. retry in 1.392275024s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:42.757225959Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\") failed: Property not found. retry in 790.826521ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:42.771121181Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\") failed: Property not found. retry in 1.455894202s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:42.784481369Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\") failed: Property not found. retry in 619.443032ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:42.797218974Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\") failed: Property not found. retry in 516.254047ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:42.810482552Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\") failed: Property not found. retry in 877.475311ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:42.824501394Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\") failed: Property not found. retry in 1.354529619s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:42.837984744Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\") failed: Property not found. retry in 845.988786ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:42.907873435Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\") failed: Property not found. retry in 1.279264423s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:42.921226943Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\") failed: Property not found. retry in 1.0307711s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:42.93435279Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\") failed: Property not found. retry in 873.82882ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:42.946972805Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\") failed: Property not found. retry in 1.119164029s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:42.965396653Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\") failed: Property not found. retry in 896.113372ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:43.034326998Z","hostname":"buskin","pid":1593,"zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\") failed: Property not found. retry in 1.174300011s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:43.059183684Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\") failed: Property not found. retry in 1.43665735s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:43.105168904Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\") failed: Property not found. retry in 1.24371234s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:43.136210277Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\") failed: Property not found. retry in 866.607447ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:43.30170046Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\") failed: Property not found. retry in 847.275711ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:43.344743043Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\") failed: Property not found. retry in 1.358061845s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:43.39194531Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\") failed: Property not found. retry in 1.108260847s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:43.434789292Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\") failed: Property not found. retry in 1.387662025s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:43.469589905Z","hostname":"buskin","pid":1593,"zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\") failed: Property not found. retry in 1.272198468s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:43.575374059Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\") failed: Property not found. retry in 1.401435599s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:43.718104497Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\") failed: Property not found. retry in 987.620011ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:43.776588481Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\") failed: Property not found. retry in 907.541331ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:43.848441424Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\") failed: Property not found. retry in 718.290359ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:43.954019974Z","hostname":"buskin","pid":1593,"zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\") failed: Property not found. retry in 1.078177464s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:43.983267876Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\") failed: Property not found. retry in 1.213766771s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:44.098777455Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\") failed: Property not found. retry in 568.529165ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:44.163241744Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\") failed: Property not found. retry in 1.240295638s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:44.181451771Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\") failed: Property not found. retry in 1.351133944s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:44.205206391Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\") failed: Property not found. retry in 759.380099ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:44.215927834Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\") failed: Property not found. retry in 850.799733ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:44.233449617Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\") failed: Property not found. retry in 986.590185ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:44.249780953Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\") failed: Property not found. retry in 702.962627ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:44.280434513Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\") failed: Property not found. retry in 1.43787657s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:44.400528719Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\") failed: Property not found. retry in 1.355719694s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:44.559461693Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\") failed: Property not found. retry in 899.921028ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:44.574521542Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\") failed: Property not found. retry in 1.358035489s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:44.685165482Z","hostname":"buskin","pid":1593,"zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\") failed: Property not found. retry in 1.017660914s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:44.746807115Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\") failed: Property not found. retry in 730.594167ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:44.763614684Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\") failed: Property not found. retry in 667.214207ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:44.776274168Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\") failed: Property not found. retry in 1.263265213s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:44.787825336Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\") failed: Property not found. retry in 1.304046997s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:44.86164442Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\") failed: Property not found. retry in 1.382697271s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:44.87350895Z","hostname":"buskin","pid":1593,"zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\") failed: Property not found. retry in 1.1062398s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:44.9893497Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\") failed: Property not found. retry in 908.959142ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:44.999611341Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\") failed: Property not found. retry in 745.690477ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:45.073543835Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\") failed: Property not found. retry in 995.878705ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:45.097996659Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\") failed: Property not found. retry in 779.541286ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:45.131327065Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\") failed: Property not found. retry in 678.878929ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:45.24955086Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\") failed: Property not found. retry in 769.094775ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:45.324515241Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\") failed: Property not found. retry in 635.869967ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:45.435481023Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\") failed: Property not found. retry in 1.227076331s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:45.455744372Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\") failed: Property not found. retry in 1.361917316s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:45.487529679Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\") failed: Property not found. retry in 1.208041347s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:45.506867443Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\") failed: Property not found. retry in 722.063906ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:45.567896012Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\") failed: Property not found. retry in 1.425653647s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:45.777660694Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\") failed: Property not found. retry in 909.30645ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:45.834917691Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\") failed: Property not found. retry in 1.254182571s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:45.850660603Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\") failed: Property not found. retry in 1.069783325s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:45.924480937Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\") failed: Property not found. retry in 1.06602795s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:45.93854054Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\") failed: Property not found. retry in 1.491932773s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:45.950581121Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\") failed: Property not found. retry in 1.05583897s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:45.962677702Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\") failed: Property not found. retry in 908.726138ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:46.057383469Z","hostname":"buskin","pid":1593,"zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\") failed: Property not found. retry in 560.695728ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:46.070964618Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\") failed: Property not found. retry in 1.020482719s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:46.083421592Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\") failed: Property not found. retry in 746.598923ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:46.109674996Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\") failed: Property not found. retry in 982.633216ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:46.123753509Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\") failed: Property not found. retry in 685.33686ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:46.1375447Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\") failed: Property not found. retry in 525.012727ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:46.179733958Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\") failed: Property not found. retry in 624.482847ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:46.274061172Z","hostname":"buskin","pid":1593,"zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\") failed: Property not found. retry in 1.173347477s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:46.304393701Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\") failed: Property not found. retry in 594.503224ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:46.645978189Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\") failed: Property not found. retry in 1.463003428s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:46.695592241Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\") failed: Property not found. retry in 698.590338ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:46.719521361Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\") failed: Property not found. retry in 683.667719ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:46.729425959Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\") failed: Property not found. retry in 1.25967529s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:46.758117208Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\") failed: Property not found. retry in 593.849187ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:46.837751336Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\") failed: Property not found. retry in 1.133301232s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:46.861218324Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\") failed: Property not found. retry in 1.359488698s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:46.872579281Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e\") failed: Property not found. retry in 1.346673307s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:46.913918204Z","hostname":"buskin","pid":1593,"zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\") failed: Property not found. retry in 734.347489ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:46.97961128Z","hostname":"buskin","pid":1593,"zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\") failed: Property not found. retry in 548.147666ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:47.009054233Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\") failed: Property not found. retry in 962.810929ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:47.024773296Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\") failed: Property not found. retry in 591.058475ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:47.036697086Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\") failed: Property not found. retry in 1.104022777s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:47.119523433Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817\") failed: Property not found. retry in 1.417074225s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:47.155130902Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327\") failed: Property not found. retry in 1.192055669s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:47.174235264Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4\") failed: Property not found. retry in 1.093242247s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:47.220090434Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\") failed: Property not found. retry in 638.752254ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:47.256136156Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2\") failed: Property not found. retry in 1.4002371s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:47.431575507Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc\") failed: Property not found. retry in 1.319618179s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:47.446359024Z","hostname":"buskin","pid":1593,"zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb\") failed: Property not found. retry in 1.107758053s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:47.458640626Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\") failed: Property not found. retry in 1.3716313s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:47.472161666Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a\") failed: Property not found. retry in 1.279163075s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:47.49502647Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\") failed: Property not found. retry in 924.053245ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:47.571678411Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\") failed: Property not found. retry in 836.777152ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:47.644237187Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\") failed: Property not found. retry in 514.964433ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:47.727588757Z","hostname":"buskin","pid":1593,"zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4\") failed: Property not found. retry in 1.095517929s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:47.911704289Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818\") failed: Property not found. retry in 600.8106ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:48.002505493Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf\") failed: Property not found. retry in 1.00205709s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:48.015073067Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\") failed: Property not found. retry in 1.198211457s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:48.03606807Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396\") failed: Property not found. retry in 1.076190424s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:48.180234998Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37\") failed: Property not found. retry in 1.478472614s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:48.242485494Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd\") failed: Property not found. retry in 565.070747ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:48.260340709Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\") failed: Property not found. retry in 1.448455242s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:48.282792481Z","hostname":"buskin","pid":1593,"zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\") failed: Property not found. retry in 522.957071ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:48.502611003Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d\") failed: Property not found. retry in 1.343813151s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:48.517040888Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944\") failed: Property not found. retry in 523.418327ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:48.91216953Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3\") failed: Property not found. retry in 763.85971ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:49.186495003Z","hostname":"buskin","pid":1593,"zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5\") failed: Property not found. retry in 664.940491ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:49.26099639Z","hostname":"buskin","pid":1593,"zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\") failed: Property not found. retry in 621.184926ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:49.820602389Z","hostname":"buskin","pid":1593,"zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620\") failed: Property not found. retry in 830.297071ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:36:50.463527699Z","hostname":"buskin","pid":1593,"zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.335872627Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_4178aff7-3700-4e3f-bdd7-968efe7848fc"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.335925458Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_63642991-e779-4349-8697-9bfc6b96d7e4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.335930668Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_external_dns_ec283d93-6725-4976-bed2-c328ce4398e5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.335935448Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_d0b8200e-a07a-4794-b87b-17a74dc261e4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.335940078Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_99ae6502-7f59-4787-8a1e-0923522db0a2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.335944258Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_external_dns_8b17fb63-5764-4496-bed6-acd4016c4f2d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.335948128Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_clickhouse_8c94f186-c252-4046-8d01-e8c5cf6a3620"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.335951558Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_f18a2ae7-5a47-47b8-9eb2-b47a19a2f818"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.335957968Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_63dcd254-bbd3-489b-9df5-0db1a21b2d37"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.335961868Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_fe5d8209-53c4-40d5-afba-39343d7ed6c3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.335965998Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_46326447-e170-4123-8268-47a9a79585dd"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.335969618Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_f05d09e0-db61-4cdb-9dbd-1b8480c392bf"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.335973178Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_39fc7bb0-36ee-45b1-96e7-097a2d543d2a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.335977388Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_oximeter_8ab3d3ec-f019-4cb7-bf89-aac78cf3c87e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.335981058Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_cfa9067d-2711-42a0-944b-e859db1c0817"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.335997858Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_44fe13c7-2d3b-4321-9d0e-0f378ee54327"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.336001778Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_10edf729-c77a-4f8c-9d40-496c39338dcb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.336006388Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_109e60dc-df19-44bb-8891-e91ae9e42944"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.336024248Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_b8f204fd-3d76-4652-963a-9ee349d4a396"} {"msg":"Writing ledger to /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.337942019Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.340265013Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.344022415Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"PUT","req_id":"9008cab9-6326-4907-a1a9-c3fd5c6cde9b","remote_addr":"[fd00:1122:3344:101::1]:64762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":39654677,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.375360729Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1212"} {"msg":"Writing ledger to /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.375946873Z","hostname":"buskin","pid":1593,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.376154614Z","hostname":"buskin","pid":1593,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.380064127Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:679"} {"msg":"Nexus address: [fd00:1122:3344:101::b]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.384600034Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:692"} {"msg":"rack_network_config: RackNetworkConfigV1 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV1 {\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.1.174,\n prefix: 24,\n },\n ),\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.1.100,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.450552421Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:803"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/76e0fa5a-5dcb-4021-ab9c-640c3670ef01/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.456476096Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/76e0fa5a-5dcb-4021-ab9c-640c3670ef01/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.545742771Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/76e0fa5a-5dcb-4021-ab9c-640c3670ef01/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.685109119Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/76e0fa5a-5dcb-4021-ab9c-640c3670ef01/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.766423148Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/76e0fa5a-5dcb-4021-ab9c-640c3670ef01/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:51.869746195Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/76e0fa5a-5dcb-4021-ab9c-640c3670ef01/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:52.067044965Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/76e0fa5a-5dcb-4021-ab9c-640c3670ef01/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:52.364505444Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/76e0fa5a-5dcb-4021-ab9c-640c3670ef01/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:52.541895806Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/76e0fa5a-5dcb-4021-ab9c-640c3670ef01/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:52.852796353Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/76e0fa5a-5dcb-4021-ab9c-640c3670ef01/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:53.351456485Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/76e0fa5a-5dcb-4021-ab9c-640c3670ef01/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:53.905931053Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/76e0fa5a-5dcb-4021-ab9c-640c3670ef01/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:54.222317173Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/76e0fa5a-5dcb-4021-ab9c-640c3670ef01/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:54.81889195Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/76e0fa5a-5dcb-4021-ab9c-640c3670ef01/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:55.558567228Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/76e0fa5a-5dcb-4021-ab9c-640c3670ef01/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:56.657178357Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/76e0fa5a-5dcb-4021-ab9c-640c3670ef01/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:58.104574224Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/76e0fa5a-5dcb-4021-ab9c-640c3670ef01/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:36:59.724611966Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"980e3d9d-7047-4041-88cc-9924138d031d\", \"content-length\": \"123\", \"date\": \"Wed, 08 May 2024 03:37:00 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"rack not found\", request_id: \"980e3d9d-7047-4041-88cc-9924138d031d\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:00.88030166Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1b40eba8-2475-4fff-a029-875712106e31\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:37:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1b40eba8-2475-4fff-a029-875712106e31\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:02.476463812Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ea6d5b55-ba70-46c2-b186-306625a75b0e\", \"content-length\": \"124\", \"date\": \"Wed, 08 May 2024 03:37:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ea6d5b55-ba70-46c2-b186-306625a75b0e\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:05.432533308Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:06.987500489Z","hostname":"buskin","pid":1593,"component":"NexusNotifierTask","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:07.124790026Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:35256"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:07.126223345Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:07.218418217Z","hostname":"buskin","pid":1593,"uri":"/inventory","method":"GET","req_id":"3b403a0e-3a56-4c29-89b0-982cb15ec4a2","remote_addr":"[fd00:1122:3344:101::a]:35256","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":93518,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:07.218495047Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:56536"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:07.21904974Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"2e819326-358f-437a-a7a3-dd982729e34c","remote_addr":"[fd00:1122:3344:101::b]:56536","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":461,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:07.219259282Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:07.219492563Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:07.221519354Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:07.221858346Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"GET","req_id":"26331448-1687-475e-b01e-41a04a7917bc","remote_addr":"[fd00:1122:3344:101::a]:35256","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3152,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:07.47589861Z","hostname":"buskin","pid":1593,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49569"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:07.476574584Z","hostname":"buskin","pid":1593,"uri":"/baseboard","method":"GET","req_id":"e63a6344-7edc-49a1-8c42-d72641ac5145","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49569","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":58,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:08.368209605Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:41998"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:08.372142928Z","hostname":"buskin","pid":1593,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6aac3518-7da3-4a4d-8b34-c4cfc1e83da8, kind: Service { id: 8b17fb63-5764-4496-bed6-acd4016c4f2d }, name: Name(\"external-dns-8b17fb63-5764-4496-bed6-acd4016c4f2d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 210])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b7228daa-3ec3-4bd0-8f53-26d1e2f21a08, kind: Service { id: ec283d93-6725-4976-bed2-c328ce4398e5 }, name: Name(\"external-dns-ec283d93-6725-4976-bed2-c328ce4398e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 115])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 48680075-02d3-4e9c-afb5-2cb5e59f4ff4, kind: Service { id: 4178aff7-3700-4e3f-bdd7-968efe7848fc }, name: Name(\"nexus-4178aff7-3700-4e3f-bdd7-968efe7848fc\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 57ab3fcc-eed9-42b4-9966-1c2fd43d1479, kind: Service { id: cfa9067d-2711-42a0-944b-e859db1c0817 }, name: Name(\"nexus-cfa9067d-2711-42a0-944b-e859db1c0817\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 68a555c7-fa8f-4574-a7ca-75539f197e18, kind: Service { id: 99ae6502-7f59-4787-8a1e-0923522db0a2 }, name: Name(\"nexus-99ae6502-7f59-4787-8a1e-0923522db0a2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 158])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:08.372779612Z","hostname":"buskin","pid":1593,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:08.372821532Z","hostname":"buskin","pid":1593,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:08.372831403Z","hostname":"buskin","pid":1593,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:08.372839243Z","hostname":"buskin","pid":1593,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:08.373328755Z","hostname":"buskin","pid":1593,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:08.373340726Z","hostname":"buskin","pid":1593,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:08.373546927Z","hostname":"buskin","pid":1593,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"e59edf9f-ff5b-408d-be08-119ee4641477","remote_addr":"[fd00:1122:3344:101::b]:41998","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":5294,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:08.374379842Z","hostname":"buskin","pid":1593,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:865"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:08.700469929Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:64737"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:08.700541119Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:08.787842422Z","hostname":"buskin","pid":1593,"uri":"/inventory","method":"GET","req_id":"a8e3ddb7-92df-4aed-99c2-f7c2a6cc0e85","remote_addr":"[fd00:1122:3344:101::b]:64737","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":87276,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:08.788387575Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:08.788463326Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:08.788835428Z","hostname":"buskin","pid":1593,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:08.78908009Z","hostname":"buskin","pid":1593,"uri":"/omicron-zones","method":"GET","req_id":"5cb3f53b-30da-44fb-9297-cbe97b57681a","remote_addr":"[fd00:1122:3344:101::b]:64737","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":943,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:30.904496127Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:63035"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:30.986084856Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"0eb975f8-f708-4df1-b4c1-f8b06450349a","remote_addr":"[fd00:1122:3344:101::c]:63035","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":81534,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:30.986233007Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:58372"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:31.039118978Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"e8795b12-56d4-49f3-ae5b-baccf389adf3","remote_addr":"[fd00:1122:3344:101::b]:58372","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":52787,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:31.03934292Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"dad43c2b-4b8a-4f88-8613-93f0492f6343","remote_addr":"[fd00:1122:3344:101::c]:63035","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":93,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:31.039613131Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"577793d6-c79f-4a5f-87e2-0854c63524d6","remote_addr":"[fd00:1122:3344:101::b]:58372","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":138,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:31.062742827Z","hostname":"buskin","pid":1593,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:31.062901158Z","hostname":"buskin","pid":1593,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:31.063925014Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:43680"} {"msg":"Writing ledger to /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:31.064066455Z","hostname":"buskin","pid":1593,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:31.06827777Z","hostname":"buskin","pid":1593,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Attempting to update network config with generation: 3, current_generation: 2","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:31.06839586Z","hostname":"buskin","pid":1593,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:31.06840471Z","hostname":"buskin","pid":1593,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:31.068476561Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"PUT","req_id":"bdf0bfc7-a1f1-4bf9-bb5f-e6b5aeb71e2f","remote_addr":"[fd00:1122:3344:101::c]:63035","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":6165,"response_code":"204"} {"msg":"Writing ledger to /pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:31.068591191Z","hostname":"buskin","pid":1593,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 3","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:31.068748692Z","hostname":"buskin","pid":1593,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:31.068800113Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"PUT","req_id":"3570f5ff-f6e8-4be8-8ac6-a06799de9575","remote_addr":"[fd00:1122:3344:101::b]:58372","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1240,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:31.109937464Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"664311f2-40cb-4faf-a147-b6ba00a74d63","remote_addr":"[fd00:1122:3344:101::a]:43680","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45913,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:31.110323277Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"0634f4ea-45e0-41b4-9558-3d151264d625","remote_addr":"[fd00:1122:3344:101::a]:43680","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":146,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:37.510737739Z","hostname":"buskin","pid":1593,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45399"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:37.51094946Z","hostname":"buskin","pid":1593,"uri":"/baseboard","method":"GET","req_id":"e0298a49-29f4-40e1-968e-f7cea27c10f2","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45399","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":119,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:37:52.855410799Z","hostname":"buskin","pid":1593,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:01.011584392Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:61617"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:01.056005153Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"5eb40830-a35c-4a35-b109-e9eb2afe6ddb","remote_addr":"[fd00:1122:3344:101::b]:61617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44216,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:01.056294215Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"c471b5eb-b2e5-4ab0-8ae3-78d170ae42a5","remote_addr":"[fd00:1122:3344:101::b]:61617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:01.151158163Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:44336"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:01.197422564Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"e9ffb556-fdc3-412f-9260-8f2ae3647f08","remote_addr":"[fd00:1122:3344:101::c]:44336","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46148,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:01.197686046Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"0a5edbc5-e664-4a3a-82a1-887fe5a59cc4","remote_addr":"[fd00:1122:3344:101::c]:44336","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:01.372063101Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:39529"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:01.41948602Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"90bbc7ed-2bd9-4bf7-8ddf-c2ac939388ef","remote_addr":"[fd00:1122:3344:101::a]:39529","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47317,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:01.419775182Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"954b6af9-f374-423f-af0b-2ac3ea2c7f71","remote_addr":"[fd00:1122:3344:101::a]:39529","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:01.449258605Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:34994","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::d]:47538"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:01.460595451Z","hostname":"buskin","pid":1593,"uri":"//411336b5-dbaf-48b1-93b3-a95d20f0f732","method":"GET","req_id":"e01d8253-480d-452b-a891-d9ed98b305d6","remote_addr":"[fd00:1122:3344:101::d]:47538","local_addr":"[fd00:1122:3344:101::1]:34994","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9947,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:07.544982987Z","hostname":"buskin","pid":1593,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35751"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:07.545084267Z","hostname":"buskin","pid":1593,"uri":"/baseboard","method":"GET","req_id":"c39aed6a-8516-45ea-885a-24c3f7530158","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35751","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":74,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:30.869037907Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:40172"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:30.919052561Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"2c1f1cca-90b1-4d0c-a3c1-63bc93f673ab","remote_addr":"[fd00:1122:3344:101::b]:40172","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49939,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:30.919209602Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:40527"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:30.922707352Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"db71312d-c157-44aa-9667-287088b42dcd","remote_addr":"[fd00:1122:3344:101::b]:40172","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3244,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:30.967272964Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"22324d9c-2282-41f0-9ba4-ff3493d4c520","remote_addr":"[fd00:1122:3344:101::c]:40527","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47932,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:30.967702547Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"b0deddb7-63b8-42bf-92aa-7cdd319cb90f","remote_addr":"[fd00:1122:3344:101::c]:40527","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":152,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:31.221028206Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:41023"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:31.270519577Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"b502e07e-d705-4842-8ca2-7f79708804f6","remote_addr":"[fd00:1122:3344:101::a]:41023","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49417,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:31.270767428Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"f70909bd-f44b-4ccd-a08f-b5dc825e15d2","remote_addr":"[fd00:1122:3344:101::a]:41023","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":95,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:31.450465265Z","hostname":"buskin","pid":1593,"uri":"//411336b5-dbaf-48b1-93b3-a95d20f0f732","method":"GET","req_id":"66bb7598-55f8-4191-ad2b-28ab1de13326","remote_addr":"[fd00:1122:3344:101::d]:47538","local_addr":"[fd00:1122:3344:101::1]:34994","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1324,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:37.579069071Z","hostname":"buskin","pid":1593,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60903"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:38:37.579166191Z","hostname":"buskin","pid":1593,"uri":"/baseboard","method":"GET","req_id":"ce24b3f5-1f56-41f2-bc24-750dcb9ca402","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60903","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":71,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:39:00.992621467Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:49636"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:39:01.043160075Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"8caf4a2a-1f53-46a8-9d3c-48d9e27efacc","remote_addr":"[fd00:1122:3344:101::b]:49636","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50458,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:39:01.043472997Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"cf1ac6e7-e5cd-409c-a92a-41fb87b82bd6","remote_addr":"[fd00:1122:3344:101::b]:49636","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:39:01.061361702Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:51909"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:39:01.106442887Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"7ee17b2d-c7b8-4f9b-b30e-e4da1b50d1ed","remote_addr":"[fd00:1122:3344:101::c]:51909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44975,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:39:01.106697788Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"06ff2ce9-70c8-4865-a99b-7a166013cbc6","remote_addr":"[fd00:1122:3344:101::c]:51909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":94,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:39:01.304711245Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:65112"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:39:01.352432473Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"52b37d6b-1d28-4351-91d2-9632ec8802f5","remote_addr":"[fd00:1122:3344:101::a]:65112","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47654,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:39:01.353284648Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"de123faf-d5c6-4136-9e1c-201de98c4ce4","remote_addr":"[fd00:1122:3344:101::a]:65112","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":694,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:39:01.450436339Z","hostname":"buskin","pid":1593,"uri":"//411336b5-dbaf-48b1-93b3-a95d20f0f732","method":"GET","req_id":"4ee882be-aefd-41ad-98bb-dca3eddf7ec8","remote_addr":"[fd00:1122:3344:101::d]:47538","local_addr":"[fd00:1122:3344:101::1]:34994","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":799,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:39:07.613620836Z","hostname":"buskin","pid":1593,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48484"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:39:07.613703507Z","hostname":"buskin","pid":1593,"uri":"/baseboard","method":"GET","req_id":"0e0eb5ec-2c86-420a-8853-14252aa1df73","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48484","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":71,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:39:30.863761023Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:48520"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:39:30.916921206Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"97bcc445-18d2-42cb-9984-85f5e1de6df8","remote_addr":"[fd00:1122:3344:101::a]:48520","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":53061,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:39:30.918083443Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"8244fa3e-d793-476f-98e8-25ef52270b7c","remote_addr":"[fd00:1122:3344:101::a]:48520","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":947,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:39:30.918506355Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:64213"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:39:30.964280834Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"bb6e3aea-95c7-49a7-b254-ae0a96412781","remote_addr":"[fd00:1122:3344:101::b]:64213","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45655,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:39:30.964542316Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"2aa09955-63df-41db-b741-4948de0fcac5","remote_addr":"[fd00:1122:3344:101::b]:64213","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":91,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:39:30.98400475Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:49013"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:39:31.032453135Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"258cb2ad-e2fc-4ae2-a754-145d64e83c6e","remote_addr":"[fd00:1122:3344:101::c]:49013","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48339,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:39:31.032825837Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"4e941d8e-8d99-4477-88fc-814ec87e6146","remote_addr":"[fd00:1122:3344:101::c]:49013","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":113,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:39:31.451168597Z","hostname":"buskin","pid":1593,"uri":"//411336b5-dbaf-48b1-93b3-a95d20f0f732","method":"GET","req_id":"f3e82209-e0e9-4fee-9028-c139f19569c2","remote_addr":"[fd00:1122:3344:101::d]:47538","local_addr":"[fd00:1122:3344:101::1]:34994","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":714,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:39:37.648192674Z","hostname":"buskin","pid":1593,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61436"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:39:37.648296315Z","hostname":"buskin","pid":1593,"uri":"/baseboard","method":"GET","req_id":"00739959-aec0-4fce-b561-7ff6d9023d5b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61436","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":75,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:40:00.990416994Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:40924"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:40:01.038744407Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"00715f35-24c8-49a8-80d1-512e06e53ca2","remote_addr":"[fd00:1122:3344:101::a]:40924","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48245,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:40:01.038975889Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"8a818061-4a9b-46bf-8a58-91835b946d6f","remote_addr":"[fd00:1122:3344:101::a]:40924","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:40:01.16002366Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:55179"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:40:01.206385443Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"e0ba1a61-8cbb-462d-b305-b052b09922ac","remote_addr":"[fd00:1122:3344:101::b]:55179","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46250,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:40:01.206649435Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"98e6a134-9add-42d6-89a6-05010bb82146","remote_addr":"[fd00:1122:3344:101::b]:55179","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:40:01.239737919Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:63864"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:40:01.282949613Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"de1b1777-f36e-41b6-b65b-92f1496a0ad2","remote_addr":"[fd00:1122:3344:101::c]:63864","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43112,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:40:01.283205195Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"310de80e-0f94-4e6d-880d-ca0082fdde49","remote_addr":"[fd00:1122:3344:101::c]:63864","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":98,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:40:01.470785787Z","hostname":"buskin","pid":1593,"uri":"//411336b5-dbaf-48b1-93b3-a95d20f0f732","method":"GET","req_id":"11d99394-2265-41de-97c7-8221c3de5e5f","remote_addr":"[fd00:1122:3344:101::d]:47538","local_addr":"[fd00:1122:3344:101::1]:34994","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":442,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:40:07.683297655Z","hostname":"buskin","pid":1593,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61748"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:40:07.683401976Z","hostname":"buskin","pid":1593,"uri":"/baseboard","method":"GET","req_id":"c19e9784-6379-4e5a-9c3f-c049682de726","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61748","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:40:30.794996108Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:58643"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:40:30.842329856Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"98025006-cd24-45f4-940e-88dd172ba330","remote_addr":"[fd00:1122:3344:101::b]:58643","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47269,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:40:30.842646948Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"6e645963-d456-4100-ba1d-49e1fe03949f","remote_addr":"[fd00:1122:3344:101::b]:58643","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:40:30.94000601Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:35445"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:40:30.98254855Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"da84ca75-70c2-4c89-9048-875e405263bf","remote_addr":"[fd00:1122:3344:101::a]:35445","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":42439,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:40:30.982819072Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"e88c37fe-da28-436a-90fb-40b0c12d2fd3","remote_addr":"[fd00:1122:3344:101::a]:35445","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:40:31.09318111Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:38675"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:40:31.137887863Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"8dd479a3-f04b-425f-b904-83e2e088bb19","remote_addr":"[fd00:1122:3344:101::c]:38675","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44663,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:40:31.138114845Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"1cf4ed02-2b2a-45ed-b8d1-3a6488de0b6a","remote_addr":"[fd00:1122:3344:101::c]:38675","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":75,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:40:31.515840565Z","hostname":"buskin","pid":1593,"uri":"//411336b5-dbaf-48b1-93b3-a95d20f0f732","method":"GET","req_id":"c4efb6bc-81d2-4ebb-823e-05b0e8f1053e","remote_addr":"[fd00:1122:3344:101::d]:47538","local_addr":"[fd00:1122:3344:101::1]:34994","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":2820,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:40:37.71754287Z","hostname":"buskin","pid":1593,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:40:37.717630291Z","hostname":"buskin","pid":1593,"uri":"/baseboard","method":"GET","req_id":"8115e16f-cf9f-464f-93c0-4c740cce34f8","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38169","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":82,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:41:01.048885186Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:33830"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:41:01.095658521Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"3e010485-5ff1-487b-901d-b96cdaa72883","remote_addr":"[fd00:1122:3344:101::b]:33830","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46699,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:41:01.095907372Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"b8991c53-9e35-44ff-a570-1bf126755cd5","remote_addr":"[fd00:1122:3344:101::b]:33830","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":92,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:41:01.24348187Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:42545"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:41:01.290445576Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"3f042de4-ac5d-4dc9-b5b2-73453657f98a","remote_addr":"[fd00:1122:3344:101::c]:42545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46856,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:41:01.290671597Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"a52a4f81-298a-4d98-aaad-61313fb823de","remote_addr":"[fd00:1122:3344:101::c]:42545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":91,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:41:01.515437469Z","hostname":"buskin","pid":1593,"uri":"//411336b5-dbaf-48b1-93b3-a95d20f0f732","method":"GET","req_id":"6be7e87f-a9b0-4fcc-8129-f74c36a83393","remote_addr":"[fd00:1122:3344:101::d]:47538","local_addr":"[fd00:1122:3344:101::1]:34994","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1245,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:41:01.537682989Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:50007"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:41:01.584930277Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"9b14b92f-ff20-4650-9f53-db6aae1b84cd","remote_addr":"[fd00:1122:3344:101::a]:50007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47175,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:41:01.585181639Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"2ebf0059-3cc3-4ff8-b8e0-a397fd2d61d3","remote_addr":"[fd00:1122:3344:101::a]:50007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":88,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:41:07.750626909Z","hostname":"buskin","pid":1593,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37846"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:41:07.75071632Z","hostname":"buskin","pid":1593,"uri":"/baseboard","method":"GET","req_id":"08771a7c-60d1-4485-ad98-f68dfa6994ee","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37846","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":83,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:41:30.856547751Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:54991"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:41:30.904606044Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"5d854874-2472-4b04-9f2b-5f7a966b648c","remote_addr":"[fd00:1122:3344:101::b]:54991","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47991,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:41:30.904792715Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:63343"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:41:30.908136695Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"0d0261dc-0e69-4363-9990-951430260dd5","remote_addr":"[fd00:1122:3344:101::b]:54991","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3218,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:41:30.9532059Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"3835aaef-5d6c-4cf9-80db-5be309713158","remote_addr":"[fd00:1122:3344:101::a]:63343","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48315,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:41:30.954204096Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"d76a462e-7434-488a-a560-198ba86a2585","remote_addr":"[fd00:1122:3344:101::a]:63343","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":781,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:41:30.965364891Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:60001"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:41:31.009008718Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"f02a26e7-7997-4013-b247-c9c4060d213c","remote_addr":"[fd00:1122:3344:101::c]:60001","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43571,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:41:31.009243999Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"017f0ef0-6120-4e06-b1f0-216ba6baead7","remote_addr":"[fd00:1122:3344:101::c]:60001","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":70,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:41:31.515033433Z","hostname":"buskin","pid":1593,"uri":"//411336b5-dbaf-48b1-93b3-a95d20f0f732","method":"GET","req_id":"f373956d-5403-41ef-abf4-3b8965d26f0f","remote_addr":"[fd00:1122:3344:101::d]:47538","local_addr":"[fd00:1122:3344:101::1]:34994","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":557,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:41:37.784213513Z","hostname":"buskin","pid":1593,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37550"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:41:37.784934567Z","hostname":"buskin","pid":1593,"uri":"/baseboard","method":"GET","req_id":"7bbf09b4-e6d4-4d3d-8f6d-79445e76fd7f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37550","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":722,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:01.100232368Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:34443"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:01.144258047Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"ffd3b7ef-7cff-4ac6-9dc1-a9c819a6cd10","remote_addr":"[fd00:1122:3344:101::a]:34443","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43952,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:01.144536019Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"a1db61ed-d161-4baf-95e2-4622ce4f6bec","remote_addr":"[fd00:1122:3344:101::a]:34443","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:01.190514909Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:59048"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:01.234619598Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"43bcf250-9e54-4fc8-9deb-0236bc471b41","remote_addr":"[fd00:1122:3344:101::c]:59048","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44001,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:01.23488845Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"5a396b15-b4ef-4ab0-9bfc-af0a65838c33","remote_addr":"[fd00:1122:3344:101::c]:59048","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:01.23670261Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:50888"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:01.279933065Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"09ba4e82-6162-4b42-8a7a-655fbf1f4cde","remote_addr":"[fd00:1122:3344:101::b]:50888","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43134,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:01.28091147Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"dfc23745-3baf-46b7-a88b-4dfca32b26ab","remote_addr":"[fd00:1122:3344:101::b]:50888","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":809,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:01.515001397Z","hostname":"buskin","pid":1593,"uri":"//411336b5-dbaf-48b1-93b3-a95d20f0f732","method":"GET","req_id":"872bfea9-8f9a-4be0-816f-6ab5c4a8d7a4","remote_addr":"[fd00:1122:3344:101::d]:47538","local_addr":"[fd00:1122:3344:101::1]:34994","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":625,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:07.819744218Z","hostname":"buskin","pid":1593,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47867"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:07.820511822Z","hostname":"buskin","pid":1593,"uri":"/baseboard","method":"GET","req_id":"4e8eeb3a-3e42-4626-813b-3e61969b4d65","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47867","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":747,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:10.933840482Z","hostname":"buskin","pid":1593,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:300"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:10.934077993Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:10.972006587Z","hostname":"buskin","pid":1593,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.99999397s","next_cleanup":"Instant { tv_sec: 11328, tv_nsec: 48069197 }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:30.858807465Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:47498"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:30.905774691Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"6a0b5d86-5a07-4498-a2cc-f5133727e719","remote_addr":"[fd00:1122:3344:101::c]:47498","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46904,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:30.905993052Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:48723"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:30.906689226Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"98bfcf3e-d793-4fcb-a0e5-e3c0e5ab4a7b","remote_addr":"[fd00:1122:3344:101::c]:47498","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":739,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:30.951394549Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"213e677f-4869-45c4-a691-c275f31ff38f","remote_addr":"[fd00:1122:3344:101::b]:48723","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45278,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:30.951815792Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"a20c3774-a4cb-4b25-ab71-25f058aceef1","remote_addr":"[fd00:1122:3344:101::b]:48723","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":153,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:31.034369487Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:33698"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:31.077656381Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"6743113f-8e8f-45a2-ba7f-66046f2b0d15","remote_addr":"[fd00:1122:3344:101::a]:33698","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43183,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:31.077968903Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"c723bfa4-40e1-4616-980a-3702f4be62c9","remote_addr":"[fd00:1122:3344:101::a]:33698","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":105,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:31.515712707Z","hostname":"buskin","pid":1593,"uri":"//411336b5-dbaf-48b1-93b3-a95d20f0f732","method":"GET","req_id":"161b845f-32a2-454e-a6ad-babe5c57e00a","remote_addr":"[fd00:1122:3344:101::d]:47538","local_addr":"[fd00:1122:3344:101::1]:34994","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":571,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:37.8533003Z","hostname":"buskin","pid":1593,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57234"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:37.85339544Z","hostname":"buskin","pid":1593,"uri":"/baseboard","method":"GET","req_id":"18035d69-e11a-4b4f-929a-d16cc99741da","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57234","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":93,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:42:52.855690975Z","hostname":"buskin","pid":1593,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:01.157822566Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:34738"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:01.201854075Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"52799a91-fe20-440c-9b85-991fe6cd5c19","remote_addr":"[fd00:1122:3344:101::b]:34738","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43971,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:01.202139727Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:34770"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:01.202362438Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"90ef636c-bfa6-4a99-b022-ca6c53fa4981","remote_addr":"[fd00:1122:3344:101::b]:34738","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":261,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:01.248867991Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"9a2ba46f-0a00-4a99-8766-39e2e0631423","remote_addr":"[fd00:1122:3344:101::c]:34770","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46573,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:01.249177293Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"cf9b6adc-19ef-413c-a6c8-f52985ac5642","remote_addr":"[fd00:1122:3344:101::c]:34770","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:01.261302635Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:52600"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:01.300733586Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"3de36649-6428-446e-87b6-644e1ff502a7","remote_addr":"[fd00:1122:3344:101::a]:52600","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":39362,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:01.301013358Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"7a72558f-9d45-4b7c-a04e-99ba788127df","remote_addr":"[fd00:1122:3344:101::a]:52600","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":100,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:01.645289122Z","hostname":"buskin","pid":1593,"uri":"//411336b5-dbaf-48b1-93b3-a95d20f0f732","method":"GET","req_id":"66837757-75b4-4eb7-b7a5-3be530b76e70","remote_addr":"[fd00:1122:3344:101::d]:47538","local_addr":"[fd00:1122:3344:101::1]:34994","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":603,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:07.887710456Z","hostname":"buskin","pid":1593,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62464"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:07.887799406Z","hostname":"buskin","pid":1593,"uri":"/baseboard","method":"GET","req_id":"a0f21680-8619-4a29-9d50-4ad692cf4637","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62464","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":83,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:30.806035953Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:38901"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:30.857019413Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"1ad64204-5757-4afd-a56d-2118b6fea574","remote_addr":"[fd00:1122:3344:101::a]:38901","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50899,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:30.857297065Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:56539"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:30.857541616Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"8cf19eb5-ef5b-49d4-995e-8825e510c6c8","remote_addr":"[fd00:1122:3344:101::a]:38901","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":234,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:30.904233371Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"c1ab6b13-38dd-4461-98c2-2ff5244e24a2","remote_addr":"[fd00:1122:3344:101::b]:56539","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46814,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:30.904574393Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"fd0802e9-7048-46d8-b785-c947258f68c4","remote_addr":"[fd00:1122:3344:101::b]:56539","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":124,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:30.960673102Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:39539"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:31.005182674Z","hostname":"buskin","pid":1593,"uri":"/switch-ports","method":"POST","req_id":"db299c03-92f9-4491-b9b8-a94edf846f8a","remote_addr":"[fd00:1122:3344:101::c]:39539","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44427,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:31.005465806Z","hostname":"buskin","pid":1593,"uri":"/network-bootstore-config","method":"GET","req_id":"05866559-be75-40e5-a5dd-61b8ed4741e5","remote_addr":"[fd00:1122:3344:101::c]:39539","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":100,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:31.646456254Z","hostname":"buskin","pid":1593,"uri":"//411336b5-dbaf-48b1-93b3-a95d20f0f732","method":"GET","req_id":"56cd355c-0e32-4f5f-b02e-9cd5b48735bf","remote_addr":"[fd00:1122:3344:101::d]:47538","local_addr":"[fd00:1122:3344:101::1]:34994","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"411336b5-dbaf-48b1-93b3-a95d20f0f732","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1623,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:32.154688861Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:57706"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:32.166327799Z","hostname":"buskin","pid":1593,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:521","metadata":"InstanceMetadata { silo_id: 8dac2178-2bd9-4d3d-ae2d-15851d0211c3, project_id: 6120e812-bc0d-472f-9b7c-f86cb8218397 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-05-08T03:43:31.700635Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(ef94a682-3038-4280-bef1-5a995ac28fbd), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-08T03:43:31.727598140Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 3cfb1958-3aef-42f5-bea9-40aca6841824, kind: Instance { id: bda7dea9-9e54-480f-acc5-e5575f71172d }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 249, 215, 108])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(3235028), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3cfb1958-3aef-42f5-bea9-40aca6841824, kind: Instance { id: bda7dea9-9e54-480f-acc5-e5575f71172d }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 249, 215, 108])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(3235028), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3cfb1958-3aef-42f5-bea9-40aca6841824, kind: Instance { id: bda7dea9-9e54-480f-acc5-e5575f71172d }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 249, 215, 108])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(3235028), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(3235028))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3cfb1958-3aef-42f5-bea9-40aca6841824, kind: Instance { id: bda7dea9-9e54-480f-acc5-e5575f71172d }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 249, 215, 108])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(3235028), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-afd9851865e\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: e8cbe33e-b6b0-4ad4-96b6-1544ea31e55b, read_only_parent: Some(Volume { block_size: 512, id: 2728540c-b0df-45e1-941b-f3cc5a445341, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 738959c6-c771-4d96-955e-1d1120abe8a6, key: Some(\"ROJ+L4dC8b6WoTWPPXqMnTdQiJo6WFiQzGfCLsVVwrk=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::12]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: e8cbe33e-b6b0-4ad4-96b6-1544ea31e55b, key: Some(\"ENl72cpprsNbdNCXbNY1JquTakKEw0CETtyguYvYHtY=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::16]:19002\", \"[fd00:1122:3344:101::15]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAALcB2oWKhYAABwHahYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAC3AdqFioWAAAcB2oWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJiZGE3ZGVhOS05ZTU0LTQ4MGYtYWNjNS1lNTU3NWY3MTE3MmQiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRExTY1lsdVAvd2RCVCs3L0FJK1M5MW9UbU95WFpTOUVkODJyTDErclhobCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"ef94a682-3038-4280-bef1-5a995ac28fbd","instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:32.16644034Z","hostname":"buskin","pid":1593,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:555","instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:32.166832102Z","hostname":"buskin","pid":1593,"component":"InstanceManager","file":"sled-agent/src/instance.rs:938","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 3cfb1958-3aef-42f5-bea9-40aca6841824, kind: Instance { id: bda7dea9-9e54-480f-acc5-e5575f71172d }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 249, 215, 108])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(3235028), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3cfb1958-3aef-42f5-bea9-40aca6841824, kind: Instance { id: bda7dea9-9e54-480f-acc5-e5575f71172d }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 249, 215, 108])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(3235028), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3cfb1958-3aef-42f5-bea9-40aca6841824, kind: Instance { id: bda7dea9-9e54-480f-acc5-e5575f71172d }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 249, 215, 108])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(3235028), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(3235028))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3cfb1958-3aef-42f5-bea9-40aca6841824, kind: Instance { id: bda7dea9-9e54-480f-acc5-e5575f71172d }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 249, 215, 108])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(3235028), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-afd9851865e\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: e8cbe33e-b6b0-4ad4-96b6-1544ea31e55b, read_only_parent: Some(Volume { block_size: 512, id: 2728540c-b0df-45e1-941b-f3cc5a445341, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 738959c6-c771-4d96-955e-1d1120abe8a6, key: Some(\"ROJ+L4dC8b6WoTWPPXqMnTdQiJo6WFiQzGfCLsVVwrk=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::12]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: e8cbe33e-b6b0-4ad4-96b6-1544ea31e55b, key: Some(\"ENl72cpprsNbdNCXbNY1JquTakKEw0CETtyguYvYHtY=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::16]:19002\", \"[fd00:1122:3344:101::15]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAALcB2oWKhYAABwHahYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAC3AdqFioWAAAcB2oWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJiZGE3ZGVhOS05ZTU0LTQ4MGYtYWNjNS1lNTU3NWY3MTE3MmQiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRExTY1lsdVAvd2RCVCs3L0FJK1M5MW9UbU95WFpTOUVkODJyTDErclhobCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(ef94a682-3038-4280-bef1-5a995ac28fbd), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-08T03:43:31.727598140Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-05-08T03:43:31.700635Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"ef94a682-3038-4280-bef1-5a995ac28fbd","instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:32.167079544Z","hostname":"buskin","pid":1593,"uri":"/instances/bda7dea9-9e54-480f-acc5-e5575f71172d","method":"PUT","req_id":"9994b976-2407-46df-9779-7cbf4f5a214f","remote_addr":"[fd00:1122:3344:101::a]:57706","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":12344,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:32.282273061Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:35581"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:32.28376916Z","hostname":"buskin","pid":1593,"uri":"/instances/bda7dea9-9e54-480f-acc5-e5575f71172d/state","method":"PUT","req_id":"d284c329-5af2-4c71-98c6-7ea48a5265fd","remote_addr":"[fd00:1122:3344:101::a]:35581","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1413,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:32.303860718Z","hostname":"buskin","pid":1593,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 249, 215, 108]), slot: 0, vni: Vni { inner: 3235028 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:32.303956098Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:32.338122259Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:32.359619956Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:33.606708666Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:33.606944808Z","hostname":"buskin","pid":1593,"zone":"oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd","instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd\") failed: Property not found. retry in 42.066444ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:43:33.779756893Z","hostname":"buskin","pid":1593,"zone":"oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd","instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd\") failed: Property not found. retry in 145.927872ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:43:33.83196128Z","hostname":"buskin","pid":1593,"zone":"oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd","instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd\") failed: Property not found. retry in 153.385655ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:43:34.023426415Z","hostname":"buskin","pid":1593,"zone":"oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd","instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd\") failed: Property not found. retry in 442.544438ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:43:34.186516374Z","hostname":"buskin","pid":1593,"zone":"oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd","instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd\") failed: Property not found. retry in 530.392116ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:43:34.640938346Z","hostname":"buskin","pid":1593,"zone":"oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd","instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd\") failed: Property not found. retry in 1.026767904s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:43:35.182015535Z","hostname":"buskin","pid":1593,"zone":"oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd","instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd\") failed: Property not found. retry in 1.422772866s","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:43:36.217826175Z","hostname":"buskin","pid":1593,"zone":"oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd","instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd\") failed: Property not found. retry in 855.592004ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:43:37.650950168Z","hostname":"buskin","pid":1593,"zone":"oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd","instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:37.920118551Z","hostname":"buskin","pid":1593,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61319"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:37.920253571Z","hostname":"buskin","pid":1593,"uri":"/baseboard","method":"GET","req_id":"6389a3b4-46f3-42c7-adbe-a0b188d9adeb","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61319","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":87,"response_code":"200"} {"msg":"Started propolis in zone: oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:38.537753952Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"sled-agent/src/instance.rs:1423"} {"msg":"wait for service Some(\"oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd\") failed: Property not found. retry in 39.244311ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:43:38.54766611Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd\") failed: Property not found. retry in 88.970123ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:43:38.596823959Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd\") failed: Property not found. retry in 164.39548ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:43:38.695376958Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd\") failed: Property not found. retry in 377.073053ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:43:38.869470972Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:39.257399782Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"sled-agent/src/instance.rs:1432"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:39.292097676Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"sled-agent/src/instance.rs:1446"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAALcB2oWKhYAABwHahYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAC3AdqFioWAAAcB2oWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJiZGE3ZGVhOS05ZTU0LTQ4MGYtYWNjNS1lNTU3NWY3MTE3MmQiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRExTY1lsdVAvd2RCVCs3L0FJK1M5MW9UbU95WFpTOUVkODJyTDErclhobCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-afd9851865e\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: e8cbe33e-b6b0-4ad4-96b6-1544ea31e55b, read_only_parent: Some(Volume { block_size: 512, id: 2728540c-b0df-45e1-941b-f3cc5a445341, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 738959c6-c771-4d96-955e-1d1120abe8a6, key: Some(\"ROJ+L4dC8b6WoTWPPXqMnTdQiJo6WFiQzGfCLsVVwrk=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::12]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: e8cbe33e-b6b0-4ad4-96b6-1544ea31e55b, key: Some(\"ENl72cpprsNbdNCXbNY1JquTakKEw0CETtyguYvYHtY=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::16]:19002\", \"[fd00:1122:3344:101::15]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: bda7dea9-9e54-480f-acc5-e5575f71172d, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: 6120e812-bc0d-472f-9b7c-f86cb8218397, silo_id: 8dac2178-2bd9-4d3d-ae2d-15851d0211c3 }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:39.294025298Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"sled-agent/src/instance.rs:685"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:39.406264787Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"sled-agent/src/instance.rs:687"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-05-08T03:43:39.407330894Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:39.407360674Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:39.407385164Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-05-08T03:43:39.407330894Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(ef94a682-3038-4280-bef1-5a995ac28fbd), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-08T03:43:31.727598140Z }","propolis_id":"ef94a682-3038-4280-bef1-5a995ac28fbd"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:39.407395624Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(ef94a682-3038-4280-bef1-5a995ac28fbd), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-08T03:43:31.727598140Z }, propolis_id: ef94a682-3038-4280-bef1-5a995ac28fbd, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-05-08T03:43:39.407330894Z } }","instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-05-08T03:43:39.438288186Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:39.438309236Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:39.438330836Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-05-08T03:43:39.438288186Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(ef94a682-3038-4280-bef1-5a995ac28fbd), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-08T03:43:31.727598140Z }","propolis_id":"ef94a682-3038-4280-bef1-5a995ac28fbd"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:39.438344026Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(ef94a682-3038-4280-bef1-5a995ac28fbd), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-08T03:43:31.727598140Z }, propolis_id: ef94a682-3038-4280-bef1-5a995ac28fbd, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-05-08T03:43:39.438288186Z } }","instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:49.495658144Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:61234"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:49.496818751Z","hostname":"buskin","pid":1593,"uri":"/instances/bda7dea9-9e54-480f-acc5-e5575f71172d/state","method":"PUT","req_id":"fe3c0b58-3ba9-493f-b62c-7f8bfd883b6b","remote_addr":"[fd00:1122:3344:101::a]:61234","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1124,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-05-08T03:43:49.496919782Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:49.496941632Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:49.496951422Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-05-08T03:43:49.496919782Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(ef94a682-3038-4280-bef1-5a995ac28fbd), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-08T03:43:31.727598140Z }","propolis_id":"ef94a682-3038-4280-bef1-5a995ac28fbd"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:49.496976942Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(ef94a682-3038-4280-bef1-5a995ac28fbd), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-08T03:43:31.727598140Z }, propolis_id: ef94a682-3038-4280-bef1-5a995ac28fbd, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-05-08T03:43:49.496919782Z } }","instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-05-08T03:43:49.525742541Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:49.525761641Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:49.525778661Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-05-08T03:43:49.525742541Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-08T03:43:49.525742541Z }","propolis_id":"ef94a682-3038-4280-bef1-5a995ac28fbd"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:49.525799531Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:49.525806442Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:49.526032633Z","hostname":"buskin","pid":1593,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:49.526074293Z","hostname":"buskin","pid":1593,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:437","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/80ce766d-f6eb-440d-b6a8-ad5a1011c02b/debug/bundle/zone\", \"/pool/int/b3bd066e-6f80-4f80-bc0d-64ec35b22d85/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/58eb44a6-f8c0-46ec-bcea-52bf3efa8996/crypt/debug\", \"/pool/ext/888a035b-7ecc-4eff-a005-8001c3d3ccff/crypt/debug\", \"/pool/ext/1afb2a46-5b47-46d7-9926-14b7e21c6407/crypt/debug\", \"/pool/ext/7c0211ed-cba6-48d8-8b63-a5f21383d980/crypt/debug\", \"/pool/ext/fc47eb1d-9a62-49dd-845d-82b29d12e969/crypt/debug\", \"/pool/ext/62a3be27-a8c4-4992-8657-cb4aacfdb8ab/crypt/debug\", \"/pool/ext/9edc3ed7-0367-4fc4-87ff-888719d0b682/crypt/debug\", \"/pool/ext/ed53811e-9a91-4270-bd15-bcb96ef98de1/crypt/debug\", \"/pool/ext/5d9bd5bb-8b9e-4dab-8435-b24d7c2ec621/crypt/debug\"] }","zone_name":"oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:50.193755218Z","hostname":"buskin","pid":1593,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1230","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd\", bundle_id: 0ee97c09-0623-4cf4-96da-f2e0f3789515 }, time_created: 2024-05-08T03:43:49.526846737Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_ef94a682-3038-4280-bef1-5a995ac28fbd","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:43:50.193863069Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:51.953375981Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:51.977911505Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-08T03:43:49.525742541Z }, propolis_id: ef94a682-3038-4280-bef1-5a995ac28fbd, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-05-08T03:43:51.977842795Z } }","instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:52.07402125Z","hostname":"buskin","pid":1593,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:33921"} {"msg":"unset_virtual_nic_host unimplmented","v":0,"name":"SledAgent","level":40,"time":"2024-05-08T03:43:52.075392648Z","hostname":"buskin","pid":1593,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:52.075413028Z","hostname":"buskin","pid":1593,"uri":"/v2p/3cfb1958-3aef-42f5-bea9-40aca6841824","method":"DELETE","req_id":"a10455df-db67-40cd-be7d-d67582046cab","remote_addr":"[fd00:1122:3344:101::a]:33921","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1240,"response_code":"204"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:52.356500651Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-08T03:43:49.525742541Z }, propolis_id: ef94a682-3038-4280-bef1-5a995ac28fbd, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-05-08T03:43:51.977842795Z } }","instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-05-08T03:43:52.359549209Z","hostname":"buskin","pid":1593,"instance_id":"bda7dea9-9e54-480f-acc5-e5575f71172d","component":"InstanceManager","file":"sled-agent/src/instance.rs:723"}