[ Apr 26 00:17:11 Enabled. ]
[ Apr 26 00:17:11 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ]
[ Apr 26 00:17:11 Method "start" exited with status 0. ]
[ Apr 26 00:17:11 Rereading configuration. ]
[ Apr 26 00:17:11 No 'refresh' method defined.  Treating as :true. ]
note: configured to log to "/dev/stdout"
{"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:11.150695439Z","hostname":"sock","pid":1590,"file":"illumos-utils/src/running_zone.rs:163"}
{"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:11.150855571Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"}
{"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:11.204424132Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"}
{"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:11.204462423Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"}
{"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:11.320722511Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"}
{"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:11.329805146Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"}
{"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:11.338621488Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:11.9449601Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:a02a::, len: 64 }"}
{"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.081824158Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:119"}
{"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.081875269Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:131"}
{"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.08190675Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:141"}
{"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.082003161Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:162","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"}
{"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.082023991Z","hostname":"sock","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:625"}
{"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.082046552Z","hostname":"sock","pid":1590,"component":"DumpSetup-archival","file":"sled-agent/src/dump_setup.rs:291"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.082066112Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"}
{"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.085915974Z","hostname":"sock","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"}
{"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:12.085931324Z","hostname":"sock","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:674"}
{"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.085936625Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:176"}
{"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.085941354Z","hostname":"sock","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:733"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.085951885Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_0.vdev"}
{"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.086017756Z","hostname":"sock","pid":1590,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"None"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.086090477Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.086104517Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.086175708Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.098200402Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.098246222Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"c162d7a8-2ba1-425d-83b7-9a5b15f2af8a (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.19395036Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: c162d7a8-2ba1-425d-83b7-9a5b15f2af8a (zpool), kind: Internal }"}
{"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.395957105Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.395999606Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_1.vdev"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.396006386Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.396017956Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.396026696Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.396031827Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.396049517Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.404766387Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.404795257Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"47941014-d07d-4a89-a7ad-e8307b721c42 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.48962347Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: 47941014-d07d-4a89-a7ad-e8307b721c42 (zpool), kind: Internal }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.68257165Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_0.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682616411Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c162d7a8-2ba1-425d-83b7-9a5b15f2af8a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 47941014-d07d-4a89-a7ad-e8307b721c42 (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682664852Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682679482Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682713193Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_1.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682737683Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682744673Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c162d7a8-2ba1-425d-83b7-9a5b15f2af8a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 47941014-d07d-4a89-a7ad-e8307b721c42 (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682757523Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682763683Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_2.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682770003Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c162d7a8-2ba1-425d-83b7-9a5b15f2af8a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 47941014-d07d-4a89-a7ad-e8307b721c42 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682798834Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682803544Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682809224Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_3.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682819064Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c162d7a8-2ba1-425d-83b7-9a5b15f2af8a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 47941014-d07d-4a89-a7ad-e8307b721c42 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682836864Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682841435Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682847765Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_4.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682854985Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682859115Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682865125Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c162d7a8-2ba1-425d-83b7-9a5b15f2af8a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 47941014-d07d-4a89-a7ad-e8307b721c42 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682883025Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_5.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682893725Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682906676Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682912036Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c162d7a8-2ba1-425d-83b7-9a5b15f2af8a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 47941014-d07d-4a89-a7ad-e8307b721c42 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682930556Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_6.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682935756Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682941016Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682946146Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c162d7a8-2ba1-425d-83b7-9a5b15f2af8a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 47941014-d07d-4a89-a7ad-e8307b721c42 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682967637Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_7.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682972997Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682985137Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.682991157Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c162d7a8-2ba1-425d-83b7-9a5b15f2af8a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 47941014-d07d-4a89-a7ad-e8307b721c42 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.683015647Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_8.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.683022138Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.683026787Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c162d7a8-2ba1-425d-83b7-9a5b15f2af8a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 47941014-d07d-4a89-a7ad-e8307b721c42 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.683055938Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.683061338Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:91"}
{"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.683072708Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:93"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.683078208Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.683082359Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c162d7a8-2ba1-425d-83b7-9a5b15f2af8a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 47941014-d07d-4a89-a7ad-e8307b721c42 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.683109359Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:200"}
{"msg":"No ledger in /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.68315183Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.68316232Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.68316851Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"}
{"msg":"No ledger in /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.68317669Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.68318473Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.68320203Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"}
{"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.683215581Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:205"}
{"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.683224281Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:220"}
{"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.694266758Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:711"}
{"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.694295659Z","hostname":"sock","pid":1590,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999916068s","next_cleanup":"Instant { tv_sec: 1514, tv_nsec: 53105425 }"}
{"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.69437033Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.69439181Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.694461051Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.697071073Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.697444629Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"}
{"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.697540301Z","hostname":"sock","pid":1590,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"}
{"msg":"Injecting RSS configuration: RackInitializeRequest {\n    trust_quorum_peers: None,\n    bootstrap_discovery: OnlyOurs,\n    ntp_servers: [\n        \"0.pool.ntp.org\",\n    ],\n    dns_servers: [\n        1.1.1.1,\n        9.9.9.9,\n    ],\n    internal_services_ip_pool_ranges: [\n        V4(\n            Ipv4Range {\n                first: 10.151.2.164,\n                last: 10.151.2.173,\n            },\n        ),\n    ],\n    external_dns_ips: [\n        10.151.2.164,\n        10.151.2.165,\n    ],\n    external_dns_zone_name: \"oxide.test\",\n    external_certificates: \"<redacted>\",\n    recovery_silo: RecoverySiloConfig {\n        silo_name: Name(\n            \"recovery\",\n        ),\n        user_name: UserId(\n            \"recovery\",\n        ),\n        user_password_hash: NewPasswordHash(\n            PasswordHashString {\n                string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n                encoding: B64,\n            },\n        ),\n    },\n    rack_network_config: RackNetworkConfigV1 {\n        rack_subnet: Ipv6Network {\n            addr: fd00:1122:3344:100::,\n            prefix: 56,\n        },\n        infra_ip_first: 10.151.2.174,\n        infra_ip_last: 10.151.2.174,\n        ports: [\n            PortConfigV1 {\n                routes: [\n                    RouteConfig {\n                        destination: V4(\n                            Ipv4Network {\n                                addr: 0.0.0.0,\n                                prefix: 0,\n                            },\n                        ),\n                        nexthop: 10.151.2.100,\n                    },\n                ],\n                addresses: [\n                    V4(\n                        Ipv4Network {\n                            addr: 10.151.2.174,\n                            prefix: 24,\n                        },\n                    ),\n                ],\n                switch: Switch0,\n                port: \"qsfp0\",\n                uplink_port_speed: Speed40G,\n                uplink_port_fec: None,\n                bgp_peers: [],\n                autoneg: false,\n            },\n        ],\n        bgp: [],\n        bfd: [],\n    },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.697754534Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:979"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.697760214Z","hostname":"sock","pid":1590,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.697812425Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.697861266Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.697881816Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.697886436Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1039"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.697908687Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.697921067Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.697935077Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.697940317Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1080"}
{"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.697943507Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"}
{"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.697946877Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.697983998Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Writing ledger to /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.697995028Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.698652109Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.698794441Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"}
{"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.698805571Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1113"}
{"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.698839782Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"}
{"msg":"Writing ledger to /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.698847432Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.698982574Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.699134566Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"}
{"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.699234978Z","hostname":"sock","pid":1590,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 779d9354-af5f-4248-aa8f-b97427c8939e, rack_id: 620b992f-d3de-45ff-bbb4-3439178676d0, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }"}
{"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.69933989Z","hostname":"sock","pid":1590,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:56324","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"}
{"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 779d9354-af5f-4248-aa8f-b97427c8939e, rack_id: 620b992f-d3de-45ff-bbb4-3439178676d0, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.699491622Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"}
{"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.699502602Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"}
{"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.699536773Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.699541643Z","hostname":"sock","pid":1590,"file":"sled-agent/src/server.rs:44"}
{"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.699563073Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:530"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.699567103Z","hostname":"sock","pid":1590,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.699572124Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"}
{"msg":"No ledger in /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.699577544Z","hostname":"sock","pid":1590,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.699583324Z","hostname":"sock","pid":1590,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.699588174Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:519"}
{"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.699592494Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:547"}
{"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.70245733Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3849"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.712159476Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.721830651Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.731065919Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.739723569Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.748804935Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.757750028Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.766369527Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.774606349Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.782870722Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.782925263Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:558"}
{"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.782930613Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:370"}
{"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.782938093Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"}
{"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.782943503Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:449","result":"DisksManagementResult { status: [] }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.782951243Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.802795742Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.820004348Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.837475499Z","hostname":"sock","pid":1590,"sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:355"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.83753317Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.83754057Z","hostname":"sock","pid":1590,"sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:367"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.83754799Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 131.321976ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.84065101Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ee6d0c84-d647-4257-bfdd-71def694b8a9\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:17:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ee6d0c84-d647-4257-bfdd-71def694b8a9\" }"}
{"msg":"attempting to create encrypted zvol: name=\"oxi_c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.850787113Z","hostname":"sock","pid":1590,"file":"sled-agent/src/swap_device.rs:153"}
{"msg":"successfully created encrypted zvol: name=\"oxi_c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.87367077Z","hostname":"sock","pid":1590,"file":"sled-agent/src/swap_device.rs:249"}
{"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.873717811Z","hostname":"sock","pid":1590,"file":"sled-agent/src/swap_device.rs:96"}
{"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.878777613Z","hostname":"sock","pid":1590,"sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:382"}
{"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.878787443Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"}
{"msg":"Ensuring dataset oxi_c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.878792633Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"}
{"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.94582057Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.972919484Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 143.634408ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:12.973576395Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6398d562-dff4-4592-b349-309f8bee581c\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:17:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6398d562-dff4-4592-b349-309f8bee581c\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.118638726Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 194.709084ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.119081203Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8026feb0-06f7-4c34-966c-da099f3500dd\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:17:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8026feb0-06f7-4c34-966c-da099f3500dd\" }"}
{"msg":"Mounting oxi_c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.163786291Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"}
{"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.177317088Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"}
{"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.1773877Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"}
{"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.17741209Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"}
{"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.177453221Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.314443022Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 212.433375ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.315053171Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a2999fff-ed28-460a-882a-c7d9650eddd4\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:17:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a2999fff-ed28-460a-882a-c7d9650eddd4\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.52843294Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 224.468689ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.529220012Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8c26816d-1e1d-47a0-8e00-66623908e8fa\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:17:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8c26816d-1e1d-47a0-8e00-66623908e8fa\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.755089591Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 329.639201ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.755821603Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1d6bfc77-b5ec-413c-a7f9-f09b78c14d79\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:17:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1d6bfc77-b5ec-413c-a7f9-f09b78c14d79\" }"}
{"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.881263688Z","hostname":"sock","pid":1590,"sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"}
{"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.927217107Z","hostname":"sock","pid":1590,"component":"VmmReservoirManager","sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"}
{"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.927265817Z","hostname":"sock","pid":1590,"component":"VmmReservoirManager","sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"}
{"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.927367509Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:993","underlay_address":"fd00:1122:3344:101::1"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.927383469Z","hostname":"sock","pid":1590,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.928449696Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"}
{"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.928491047Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.928518528Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:13.928528278Z","hostname":"sock","pid":1590,"sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1190","retry_after":"185.166398ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""}
{"msg":"Writing ledger to /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.928535958Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.928780762Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"common/src/ledger.rs:199"}
{"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.928938414Z","hostname":"sock","pid":1590,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"}
{"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.928951615Z","hostname":"sock","pid":1590,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.2.100 }], addresses: [V4(Ipv4Network { addr: 10.151.2.174, prefix: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"}
{"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.928958305Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"}
{"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:13.928998945Z","hostname":"sock","pid":1590,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:56324","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"}
{"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.122298491Z","hostname":"sock","pid":1590,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"}
{"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.122370872Z","hostname":"sock","pid":1590,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.122660787Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.122736458Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.122750838Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.122786229Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.122795939Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.122945701Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:14.123964197Z","hostname":"sock","pid":1590,"sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1190","retry_after":"524.993547ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/779d9354-af5f-4248-aa8f-b97427c8939e): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:14.128301317Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.155286851Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:55841"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.155465114Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.155542845Z","hostname":"sock","pid":1590,"uri":"/inventory","method":"GET","req_id":"2a9cf050-6688-4990-bedb-a1c0d61a0b39","remote_addr":"[fd00:1122:3344:101::1]:55841","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":156,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.185368614Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:47113"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.185465365Z","hostname":"sock","pid":1590,"uri":"/sled-role","method":"GET","req_id":"962e6269-566a-4488-989e-8b400ecec96f","remote_addr":"[fd00:1122:3344:101::1]:47113","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.186885928Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Writing ledger to /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.186919519Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.187209144Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.187482738Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:815"}
{"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: bbdda3ce-e865-4591-a829-44707d6664b3, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, pool_id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool) }, OmicronPhysicalDiskConfig { id: 5d90909d-e7ec-49b1-bb08-6292b03161b2, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, pool_id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool) }, OmicronPhysicalDiskConfig { id: c8d6fc8d-a6c9-47b8-a193-265047059622, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, pool_id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool) }, OmicronPhysicalDiskConfig { id: 51a91b4a-b0c2-42e8-ae6b-0dacae6cd00a, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, pool_id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool) }, OmicronPhysicalDiskConfig { id: 089c72c3-bec3-47f4-9bfb-5de85aab5e93, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, pool_id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool) }, OmicronPhysicalDiskConfig { id: 10dee09c-f673-4b97-9f23-e37549dce43e, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, pool_id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool) }, OmicronPhysicalDiskConfig { id: b6f05aac-d09b-482a-9293-46a659a27c32, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, pool_id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool) }, OmicronPhysicalDiskConfig { id: 61ab3e82-883e-42c9-bdf0-842c588c072d, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, pool_id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool) }, OmicronPhysicalDiskConfig { id: cc376c91-9ab3-41bc-864a-be71f25ef60c, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, pool_id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.217915337Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:346"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.217979128Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:33402"}
{"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, id: bbdda3ce-e865-4591-a829-44707d6664b3, pool_id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, id: 5d90909d-e7ec-49b1-bb08-6292b03161b2, pool_id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, id: c8d6fc8d-a6c9-47b8-a193-265047059622, pool_id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, id: 51a91b4a-b0c2-42e8-ae6b-0dacae6cd00a, pool_id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, id: 089c72c3-bec3-47f4-9bfb-5de85aab5e93, pool_id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, id: 10dee09c-f673-4b97-9f23-e37549dce43e, pool_id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, id: b6f05aac-d09b-482a-9293-46a659a27c32, pool_id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, id: 61ab3e82-883e-42c9-bdf0-842c588c072d, pool_id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, id: cc376c91-9ab3-41bc-864a-be71f25ef60c, pool_id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.218229292Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.218238362Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.218248812Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.218256952Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:681"}
{"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.218264092Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.218271352Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.218280133Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.218286893Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.302623588Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:14.302675399Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool)"}
{"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:14.677056024Z","hostname":"sock","pid":1590,"sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1190","retry_after":"581.342433ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/779d9354-af5f-4248-aa8f-b97427c8939e): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-26T00:17:15.107644631Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:15.299220709Z","hostname":"sock","pid":1590,"sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1190","retry_after":"1.672371713s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:15.30985543Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:15.502931902Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:15.503055544Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:15.503065484Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:15.503071284Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:15.503080784Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:15.503190496Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:15.503199146Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/779d9354-af5f-4248-aa8f-b97427c8939e): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:16.070771785Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:16.074039327Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/779d9354-af5f-4248-aa8f-b97427c8939e): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-26T00:17:16.264356065Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Automatically destroying dataset: oxp_d264dd09-dd1e-4b44-ac20-70d4e8728824/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:16.269832363Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:16.598261149Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 37.857356ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:17.353121147Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/779d9354-af5f-4248-aa8f-b97427c8939e): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-26T00:17:17.35452305Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:17.35456795Z","hostname":"sock","pid":1590,"sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1190","retry_after":"3.024230561s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 144.331166ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:17.939420637Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/779d9354-af5f-4248-aa8f-b97427c8939e): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:18.701163175Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/779d9354-af5f-4248-aa8f-b97427c8939e): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-26T00:17:18.701227866Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 269.108631ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:18.701428279Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Looking for unencrypted datasets in oxp_d264dd09-dd1e-4b44-ac20-70d4e8728824","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:19.411761731Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 275.581141ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:19.654996659Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:19.660192653Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:19.660211603Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:19.660226403Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:19.660236434Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:19.660248084Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:19.88350113Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:19.883552701Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"1962e339-1294-4041-a983-29e64bdb2ffe (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:21.073741952Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:21.313017027Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:21.313061337Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:21.313084588Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:21.313091898Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:21.313136699Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:21.313296261Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:21.313305372Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 765.444558ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:22.069775965Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:22.072459278Z","hostname":"sock","pid":1590,"sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1190","retry_after":"7.828699119s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/779d9354-af5f-4248-aa8f-b97427c8939e): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-26T00:17:22.072516619Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/779d9354-af5f-4248-aa8f-b97427c8939e): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:22.07260849Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:22.374790625Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_1962e339-1294-4041-a983-29e64bdb2ffe/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:22.888361426Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/779d9354-af5f-4248-aa8f-b97427c8939e): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-26T00:17:23.3553858Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/779d9354-af5f-4248-aa8f-b97427c8939e): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:23.84523034Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.545177345Z","hostname":"sock","pid":1590,"component":"VmmReservoirManager","sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/779d9354-af5f-4248-aa8f-b97427c8939e): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-26T00:17:24.547673745Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Looking for unencrypted datasets in oxp_1962e339-1294-4041-a983-29e64bdb2ffe","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.631731135Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.639941677Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.639983988Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.639992928Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.639999668Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.640008288Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.650332244Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.650379265Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"29f0e314-9945-472b-979d-b91f7fcb5170 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.746980767Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.754937175Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.754973715Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.754985106Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.755053497Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.755069127Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.755162028Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.755169749Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.791232668Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_29f0e314-9945-472b-979d-b91f7fcb5170/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.801080506Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_29f0e314-9945-472b-979d-b91f7fcb5170","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.924909956Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.933091947Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.933146868Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.933154178Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.933160358Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.933165508Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.943194959Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:24.94322555Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.038399199Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.046769553Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.046813754Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.046820814Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.046842064Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.046902775Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.047024637Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.047047707Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.084174764Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_b4f1d1aa-0185-4f19-9f65-a972831259c3/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.092229693Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_b4f1d1aa-0185-4f19-9f65-a972831259c3","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.212859122Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.220564585Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.220602976Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.220609516Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.220615636Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.220622456Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.230267161Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.230295882Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.318140853Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.326445476Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.326488237Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.326494907Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.326549088Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.326559998Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.326708921Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.326715811Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.361270306Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_fa79b281-89d1-42f4-b871-62a5aaaf9922/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.369098852Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_fa79b281-89d1-42f4-b871-62a5aaaf9922","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.493595272Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.501482179Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.501519269Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.501525879Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.501531309Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.50154941Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.51155547Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.511584151Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.607085795Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.615546131Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.615576182Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.615582362Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.615614742Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.615623573Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.615776345Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.615783355Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/779d9354-af5f-4248-aa8f-b97427c8939e): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-26T00:17:25.619921411Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/779d9354-af5f-4248-aa8f-b97427c8939e): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:25.620097034Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.651889775Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_42895196-0869-4ea3-9c4e-285bbcea1a80/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.660512174Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_42895196-0869-4ea3-9c4e-285bbcea1a80","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.789617618Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.797756709Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.797809669Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.79781976Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.79782962Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.79783836Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.80714517Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.8071835Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.895718783Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.903608869Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.9036649Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.903676161Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.903686251Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.903698811Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.903851993Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.903864473Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.938123814Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_0de19191-ee35-4d6e-ab04-5e684dafd16c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:25.946619871Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_0de19191-ee35-4d6e-ab04-5e684dafd16c","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.076427956Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.084749149Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.08479085Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.08480172Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.08480922Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.08482686Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.094230022Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.094254032Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.189537183Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.197812106Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.197848796Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.197868247Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.197913468Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.197929478Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.198151921Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.198164541Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.235635023Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.243712973Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.316735946Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2303"}
{"msg":"Looking for unencrypted datasets in oxp_1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.373047891Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.381189442Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.381230823Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.381237483Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.381242893Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.381247233Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.390685405Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.390711145Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.48749245Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.495622651Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.495649481Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.495658581Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.495671072Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.495680561Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.495861064Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.495871985Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.531470227Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_73bca2cf-2bf8-443a-bf93-681a0fe6b326/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.539181451Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/779d9354-af5f-4248-aa8f-b97427c8939e): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-26T00:17:26.621453292Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Looking for unencrypted datasets in oxp_73bca2cf-2bf8-443a-bf93-681a0fe6b326","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.666002138Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.674271091Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.674316012Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Writing ledger to /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.674382183Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.674555826Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c162d7a8-2ba1-425d-83b7-9a5b15f2af8a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 47941014-d07d-4a89-a7ad-e8307b721c42 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Writing ledger to /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.674754629Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.675142995Z","hostname":"sock","pid":1590,"uri":"/omicron-physical-disks","method":"PUT","req_id":"ba7a1649-35e0-439e-afe9-565beb56c94d","remote_addr":"[fd00:1122:3344:101::1]:33402","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":12457117,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.705767717Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.705907919Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:47597"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.706201454Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.706208284Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.759012412Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.759059473Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.810766724Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.810817755Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.862237341Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.862278162Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.862305722Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:188"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.862361973Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.907478848Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.907519338Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.907525589Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_dcbedba4-6a83-41c6-9a2a-514fb195d796","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.959856039Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_ed9071c1-21f2-4f15-812c-6b6ce9f54d7b","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.965066493Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_2459ecd0-1d68-43a3-8a7a-97b992a6728f","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.969225209Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_internal_dns_dcbedba4-6a83-41c6-9a2a-514fb195d796","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.981951894Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_internal_dns_ed9071c1-21f2-4f15-812c-6b6ce9f54d7b","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.986498327Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_internal_dns_2459ecd0-1d68-43a3-8a7a-97b992a6728f","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:26.990894877Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/779d9354-af5f-4248-aa8f-b97427c8939e): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:27.618738775Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/779d9354-af5f-4248-aa8f-b97427c8939e): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-26T00:17:27.6234076Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/779d9354-af5f-4248-aa8f-b97427c8939e): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-26T00:17:28.6255314Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Profile for oxz_internal_dns_ed9071c1-21f2-4f15-812c-6b6ce9f54d7b:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService2\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:2::2\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:2::1\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:2::1]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"[fd00:1122:3344:2::1]:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:29.289226793Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:29.289331584Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:29.289439696Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_ed9071c1-21f2-4f15-812c-6b6ce9f54d7b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_internal_dns_2459ecd0-1d68-43a3-8a7a-97b992a6728f:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService3\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:3::2\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:3::1\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:3::1]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"[fd00:1122:3344:3::1]:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:29.773967401Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:29.774028352Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"}
{"msg":"Profile for oxz_internal_dns_dcbedba4-6a83-41c6-9a2a-514fb195d796:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService1\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:1::2\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:1::1\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:1::1]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"[fd00:1122:3344:1::1]:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:30.255193712Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"wait for service Some(\"oxz_internal_dns_ed9071c1-21f2-4f15-812c-6b6ce9f54d7b\") failed: Property not found. retry in 66.52793ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:30.26258695Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_ed9071c1-21f2-4f15-812c-6b6ce9f54d7b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:30.262631141Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_2459ecd0-1d68-43a3-8a7a-97b992a6728f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:30.266656746Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:30.266692616Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_dcbedba4-6a83-41c6-9a2a-514fb195d796","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_internal_dns_ed9071c1-21f2-4f15-812c-6b6ce9f54d7b\") failed: Property not found. retry in 132.397948ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:30.339368484Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_ed9071c1-21f2-4f15-812c-6b6ce9f54d7b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_dcbedba4-6a83-41c6-9a2a-514fb195d796\") failed: Property not found. retry in 62.113404ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:30.394130884Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_dcbedba4-6a83-41c6-9a2a-514fb195d796","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_2459ecd0-1d68-43a3-8a7a-97b992a6728f\") failed: Property not found. retry in 74.974647ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:30.400932823Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_2459ecd0-1d68-43a3-8a7a-97b992a6728f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_dcbedba4-6a83-41c6-9a2a-514fb195d796\") failed: Property not found. retry in 104.731492ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:30.574070145Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_dcbedba4-6a83-41c6-9a2a-514fb195d796","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_ed9071c1-21f2-4f15-812c-6b6ce9f54d7b\") failed: Property not found. retry in 118.387113ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:30.614257761Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_ed9071c1-21f2-4f15-812c-6b6ce9f54d7b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_2459ecd0-1d68-43a3-8a7a-97b992a6728f\") failed: Property not found. retry in 145.345814ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:30.6547097Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_2459ecd0-1d68-43a3-8a7a-97b992a6728f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_dcbedba4-6a83-41c6-9a2a-514fb195d796\") failed: Property not found. retry in 255.985015ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:30.724694305Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_dcbedba4-6a83-41c6-9a2a-514fb195d796","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_ed9071c1-21f2-4f15-812c-6b6ce9f54d7b\") failed: Property not found. retry in 372.133068ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:30.757532023Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_ed9071c1-21f2-4f15-812c-6b6ce9f54d7b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_2459ecd0-1d68-43a3-8a7a-97b992a6728f\") failed: Property not found. retry in 158.046878ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:30.834275096Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_2459ecd0-1d68-43a3-8a7a-97b992a6728f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_dcbedba4-6a83-41c6-9a2a-514fb195d796\") failed: Property not found. retry in 446.908794ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:30.990551406Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_dcbedba4-6a83-41c6-9a2a-514fb195d796","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_2459ecd0-1d68-43a3-8a7a-97b992a6728f\") failed: Property not found. retry in 394.691648ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:30.999778054Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_2459ecd0-1d68-43a3-8a7a-97b992a6728f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_ed9071c1-21f2-4f15-812c-6b6ce9f54d7b\") failed: Property not found. retry in 945.346515ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:31.138604885Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_ed9071c1-21f2-4f15-812c-6b6ce9f54d7b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_2459ecd0-1d68-43a3-8a7a-97b992a6728f\") failed: Property not found. retry in 986.419701ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:31.403903437Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_2459ecd0-1d68-43a3-8a7a-97b992a6728f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_dcbedba4-6a83-41c6-9a2a-514fb195d796\") failed: Property not found. retry in 1.033785324s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:31.447017649Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_dcbedba4-6a83-41c6-9a2a-514fb195d796","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_ed9071c1-21f2-4f15-812c-6b6ce9f54d7b\") failed: Property not found. retry in 654.720341ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:32.093412714Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_ed9071c1-21f2-4f15-812c-6b6ce9f54d7b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:32.357959135Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2320"}
{"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:32.357982925Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"}
{"msg":"wait for service Some(\"oxz_internal_dns_2459ecd0-1d68-43a3-8a7a-97b992a6728f\") failed: Property not found. retry in 575.412502ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:32.400719042Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_2459ecd0-1d68-43a3-8a7a-97b992a6728f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_dcbedba4-6a83-41c6-9a2a-514fb195d796\") failed: Property not found. retry in 1.323149618s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:32.490093568Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_dcbedba4-6a83-41c6-9a2a-514fb195d796","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_ed9071c1-21f2-4f15-812c-6b6ce9f54d7b\") failed: Property not found. retry in 639.487625ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:32.75663087Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_ed9071c1-21f2-4f15-812c-6b6ce9f54d7b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fed4:7cf9","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:32.916117432Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2327"}
{"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:32.922035467Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2388"}
{"msg":"wait for service Some(\"oxz_internal_dns_2459ecd0-1d68-43a3-8a7a-97b992a6728f\") failed: Property not found. retry in 1.424844567s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:32.983576546Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_2459ecd0-1d68-43a3-8a7a-97b992a6728f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:33.044355262Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2573"}
{"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:33.296082476Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2737"}
{"msg":"wait for service Some(\"oxz_internal_dns_ed9071c1-21f2-4f15-812c-6b6ce9f54d7b\") failed: Property not found. retry in 1.29189294s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:33.404487348Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_ed9071c1-21f2-4f15-812c-6b6ce9f54d7b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:33.489127458Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2452"}
{"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:33.792131506Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2487"}
{"msg":"wait for service Some(\"oxz_internal_dns_dcbedba4-6a83-41c6-9a2a-514fb195d796\") failed: Property not found. retry in 911.292292ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:33.824257233Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_dcbedba4-6a83-41c6-9a2a-514fb195d796","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:33.992338493Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2787"}
{"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:34.167416455Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2814"}
{"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:34.397153436Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2681"}
{"msg":"wait for service Some(\"oxz_internal_dns_2459ecd0-1d68-43a3-8a7a-97b992a6728f\") failed: Property not found. retry in 986.754156ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:34.418659262Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_2459ecd0-1d68-43a3-8a7a-97b992a6728f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:34.585429531Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2481"}
{"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:34.595453082Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3873","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"}
{"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:34.595497393Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3889"}
{"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:34.595505853Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"}
{"msg":"wait for service Some(\"oxz_internal_dns_dcbedba4-6a83-41c6-9a2a-514fb195d796\") failed: Property not found. retry in 1.070009828s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:34.744634739Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_dcbedba4-6a83-41c6-9a2a-514fb195d796","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.152616854Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3897"}
{"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.159355152Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3953"}
{"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.292887247Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:4015"}
{"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.400309763Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3996","rack_subnet":"fd00:1122:3344::"}
{"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.425213613Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:4043"}
{"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.54449045Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:4070"}
{"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.683669536Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:364"}
{"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.719224197Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:412"}
{"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.752895988Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:589"}
{"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.752976839Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:627"}
{"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.753396016Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:630","uptime":"2"}
{"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.753427947Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:434","config":"PortSettings {\n    links: {\n        \"0\": LinkSettings {\n            addrs: [\n                10.151.2.174,\n            ],\n            params: LinkCreate {\n                autoneg: false,\n                fec: None,\n                kr: false,\n                lane: Some(\n                    LinkId(\n                        0,\n                    ),\n                ),\n                speed: Speed40G,\n            },\n        },\n    },\n}"}
{"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.8345469Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.834619641Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.834629491Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.834638821Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.834646011Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.834651661Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.834659222Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.834666402Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.834673492Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.834680912Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.834687352Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.834696252Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.834716673Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.834722033Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.834726963Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.834731833Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.834737243Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.834742403Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.834747333Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.834752003Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.834756693Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.834761573Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.834766663Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.83702143Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_dcbedba4-6a83-41c6-9a2a-514fb195d796"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.83703518Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_ed9071c1-21f2-4f15-812c-6b6ce9f54d7b"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.83704191Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_2459ecd0-1d68-43a3-8a7a-97b992a6728f"}
{"msg":"Writing ledger to /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.83705071Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.837342525Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.837562718Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"bb960011-f540-4014-a7a2-7f95e8e4a14d","remote_addr":"[fd00:1122:3344:101::1]:47597","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9131589,"response_code":"204"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:35.839239665Z","hostname":"sock","pid":1590,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:547"}
{"msg":"failed to write DNS configuration (will retry in 122.539438ms)","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:36.15326383Z","hostname":"sock","pid":1590,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:567","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.292382035Z","hostname":"sock","pid":1590,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:547"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.337835976Z","hostname":"sock","pid":1590,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:547"}
{"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.38414116Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:583"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.38415615Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:258"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.416134174Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.418145646Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:300"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.418157706Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.447229133Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.447401436Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:45322"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.448443573Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.448448703Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.469376019Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.46943369Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.488260912Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.488314583Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.507579023Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.507649754Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.507781576Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.507897148Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.518657401Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.549221622Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.549867002Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.549876182Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.594579541Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 200, 220]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.594621401Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"ntp"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.2.169,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.59515246Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.2.169,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.59517033Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.2.169,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.596010164Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.597873954Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.622167064Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:36.641054357Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService4\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::1b\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/chrony-setup\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"allow\" value=\"fd00:1122:3344:100::/56\"/>\n        <propval type=\"boolean\" name=\"boundary\" value=\"true\"/>\n        <propval type=\"astring\" name=\"server\" value=\"0.pool.ntp.org\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"1.1.1.1\" />\n            <value_node value=\"9.9.9.9\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/ntp\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte0\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.3.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.3.5\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:37.773250507Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:37.773383229Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0\") failed: Property not found. retry in 33.910995ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:37.906777982Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0\") failed: Property not found. retry in 62.177085ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:37.948494182Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0\") failed: Property not found. retry in 228.101451ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:38.022864587Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0\") failed: Property not found. retry in 204.416421ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:38.262158841Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0\") failed: Property not found. retry in 908.028922ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:38.476053978Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:39.080489629Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52800"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:39.080671122Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"409620ba-b405-4df3-ad8b-6b61649320f7","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52800","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":87,"response_code":"200"}
{"msg":"wait for service Some(\"oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0\") failed: Property not found. retry in 1.122798649s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:39.394642096Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0\") failed: Property not found. retry in 1.404472998s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:40.526410309Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0\") failed: Property not found. retry in 1.225295751s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:41.940173013Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:43.185611272Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0"}
{"msg":"Writing ledger to /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:43.185681253Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:43.185936127Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:43.186207451Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"ccede01c-fa74-457d-9d27-37ad9b34cb90","remote_addr":"[fd00:1122:3344:101::1]:45322","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":6738754,"response_code":"204"}
{"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:43.188276284Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:624"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:43.216928925Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:43.217107138Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:37408"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:43.225940049Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:43.226115542Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"f8440abe-fdec-449f-aade-434ebd0ee396","remote_addr":"[fd00:1122:3344:101::1]:37408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8911,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:43.228823936Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:43.366893704Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:43.367108888Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:61392"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:47.375597698Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:47.37571042Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"a013d1ea-6c15-4dee-990c-21ad05ee17c6","remote_addr":"[fd00:1122:3344:101::1]:61392","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008561,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:47.377778253Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:47.538689589Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:47.538836401Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:48002"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:47.547059483Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:47.547209235Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"aa4ea62d-a09a-49c8-a4e1-425c66e210f5","remote_addr":"[fd00:1122:3344:101::1]:48002","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8249,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:47.548921083Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:47.67695478Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:47.677083332Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:44559"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_0755d222-4cb0-4d69-ba56-feb7502debc0': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:51.685327868Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:51.68545932Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"56332993-a231-470b-9271-c1baef5dfb33","remote_addr":"[fd00:1122:3344:101::1]:44559","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008250,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:51.68856755Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:51.942598602Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:51.942790115Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:43790"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:51.951475614Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"6112df24-ab28-488f-b3e4-eabb49e03942","remote_addr":"[fd00:1122:3344:101::1]:43790","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8637,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714090666.7411778, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:51.953319104Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:51.953325534Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:52.19648454Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:52.196649333Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:46320"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:52.205880691Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"76bfac2b-d016-4a22-acbb-38ec06e21ef5","remote_addr":"[fd00:1122:3344:101::1]:46320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9123,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714090666.7411778, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:52.208373401Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:52.208381661Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:52.402723994Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:52.402821985Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:63257"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:52.410845414Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"63c10f3b-23ab-4baf-a053-3dbd84b93e03","remote_addr":"[fd00:1122:3344:101::1]:63257","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7933,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714090666.7411778, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:52.412659083Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:52.412669214Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:52.816310749Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:52.816408Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:57381"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:52.824064503Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"c3dc4835-be3b-499f-be9b-607dd4ed87bf","remote_addr":"[fd00:1122:3344:101::1]:57381","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7528,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714090666.7411778, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:52.825296213Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:52.825303793Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.461059977Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.461156399Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:60064"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.46374161Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.468390995Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"855a7be9-23ea-4972-b2ba-4c138487bfe1","remote_addr":"[fd00:1122:3344:101::1]:60064","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7151,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714090666.7411778, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.470057832Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:53.470063772Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.139845617Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.139951248Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:37355"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.142378437Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.159610364Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"5e4a8ae8-506c-4395-8d95-074dd56e4d1d","remote_addr":"[fd00:1122:3344:101::1]:37355","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":19531,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2732330510, ip_addr: 162.220.14.14, stratum: 5, ref_time: 1714090673.0336883, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.161036857Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"}
{"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.161044537Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1192"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.189894271Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.190012763Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:58406"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.190236086Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.190246717Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.20915601Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.209214411Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.226113943Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.226166514Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.244212353Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.244256274Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.296104087Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.296145187Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.348326256Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.348367476Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.400005336Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.400051667Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.451481723Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.451528194Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.502201638Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.502265869Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.502528683Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.502671045Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.58203274Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.582102832Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.582113682Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.582119622Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.582125342Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.67164158Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.675405361Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.679194792Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.683113744Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.6871958Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.692890642Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.698201447Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.703116856Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.707347904Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:53.711671003Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService8\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::3\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::3\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:57.363635095Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:57.363817607Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:57.366971118Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"}
{"msg":"Profile for oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService6\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::4\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::4\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:57.373745737Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:57.37391145Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService5\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::6\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::6\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:57.481743252Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:57.481900085Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86\") failed: Property not found. retry in 38.179811ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:57.510659547Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401\") failed: Property not found. retry in 28.90097ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:57.517861722Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401\") failed: Property not found. retry in 75.628871ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:57.555305223Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86\") failed: Property not found. retry in 86.493546ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:57.562372597Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401\") failed: Property not found. retry in 296.435177ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:57.673077335Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86\") failed: Property not found. retry in 123.78464ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:57.70821731Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService7\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::7\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::7\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:57.719246827Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService9\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::5\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::5\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:57.719295028Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:57.719940648Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:17:57.730127882Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3\") failed: Property not found. retry in 41.755528ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:57.81092482Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86\") failed: Property not found. retry in 272.598583ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:57.856076916Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3\") failed: Property not found. retry in 115.595128ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:57.870484637Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401\") failed: Property not found. retry in 372.089719ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:58.059695647Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3\") failed: Property not found. retry in 223.17709ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:58.175078131Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86\") failed: Property not found. retry in 794.012654ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:58.313774859Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137\") failed: Property not found. retry in 41.753451ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:58.345394847Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8\") failed: Property not found. retry in 41.601192ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:58.384038428Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137\") failed: Property not found. retry in 125.554236ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:58.452053491Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3\") failed: Property not found. retry in 471.928389ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:58.505438319Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401\") failed: Property not found. retry in 1.158177299s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:58.547793099Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8\") failed: Property not found. retry in 110.749974ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:58.583074886Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137\") failed: Property not found. retry in 143.888172ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:58.614583432Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8\") failed: Property not found. retry in 120.386553ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:58.812311419Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137\") failed: Property not found. retry in 438.469539ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:59.086153708Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3\") failed: Property not found. retry in 1.038119776s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:59.32155222Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86\") failed: Property not found. retry in 862.638947ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:59.404655986Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8\") failed: Property not found. retry in 342.128717ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:59.48707386Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137\") failed: Property not found. retry in 456.733307ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:59.552101164Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401\") failed: Property not found. retry in 1.44854754s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:59.74230421Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8\") failed: Property not found. retry in 912.312812ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:17:59.852925107Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137\") failed: Property not found. retry in 1.320290374s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:00.028366126Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86\") failed: Property not found. retry in 582.220538ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:00.280246283Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3\") failed: Property not found. retry in 1.457102102s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:00.370979831Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8\") failed: Property not found. retry in 1.155876703s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:00.776738409Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86\") failed: Property not found. retry in 1.122120331s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:00.871595383Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401\") failed: Property not found. retry in 1.13861389s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:01.200418866Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137\") failed: Property not found. retry in 854.091339ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:01.358568357Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3\") failed: Property not found. retry in 752.071796ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:01.839142688Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8\") failed: Property not found. retry in 1.194916691s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:01.94195447Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86\") failed: Property not found. retry in 1.263775711s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:02.003827914Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137\") failed: Property not found. retry in 813.825463ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:02.22580491Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401\") failed: Property not found. retry in 1.257215243s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:02.350504834Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3\") failed: Property not found. retry in 965.138512ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:02.603084361Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137\") failed: Property not found. retry in 554.43179ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:03.049552844Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8\") failed: Property not found. retry in 890.218564ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:03.149570241Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86\") failed: Property not found. retry in 1.259927498s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:03.277281493Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3\") failed: Property not found. retry in 1.082002222s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:03.575737567Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137\") failed: Property not found. retry in 1.20191659s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:03.611116606Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401\") failed: Property not found. retry in 579.194955ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:03.619741454Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8\") failed: Property not found. retry in 1.234091843s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:04.047879813Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401\") failed: Property not found. retry in 726.749279ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:04.20640938Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86\") failed: Property not found. retry in 519.393024ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:04.54561425Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3\") failed: Property not found. retry in 1.400098373s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:04.667968615Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137\") failed: Property not found. retry in 573.882563ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:04.824728744Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401\") failed: Property not found. retry in 973.114839ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:04.941439549Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86\") failed: Property not found. retry in 653.762714ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:05.074160641Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8\") failed: Property not found. retry in 1.413345196s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:05.291774858Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137\") failed: Property not found. retry in 1.343922134s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:05.407720391Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86\") failed: Property not found. retry in 983.325616ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:05.738352172Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:06.774337866Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_e9e24743-a540-4b93-9133-4eae210ed8f3"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:06.774411937Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_ac169011-c605-4b75-aa7b-c3144c1ddd86"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:06.774424138Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_6a642efd-fc7d-4434-bb44-b7f02c3b3137"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:06.774446478Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:06.774453968Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_8cc9856d-c4e4-49e3-9237-1d97f538eae8"}
{"msg":"Writing ledger to /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:06.774460218Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:06.774843455Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:06.77518641Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"5e31dc38-2f59-4e2c-95f8-706d1757340b","remote_addr":"[fd00:1122:3344:101::1]:58406","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":13272110,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:06.806879039Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:47419"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:06.835375327Z","hostname":"sock","pid":1590,"uri":"/cockroachdb","method":"POST","req_id":"a2193ac8-8384-4aef-822c-720786e8fa1d","remote_addr":"[fd00:1122:3344:101::1]:47419","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_5e3ff898-c77b-42a1-9764-187791149401 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1  stdout: \n  stderr: ipadm: Address object not found","latency_us":28322,"response_code":"500"}
{"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:06.835707572Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:934","retry_after":"88.606736ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a2193ac8-8384-4aef-822c-720786e8fa1d\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:18:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a2193ac8-8384-4aef-822c-720786e8fa1d\" }"}
{"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:06.956753607Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3437"}
{"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:17.312013745Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3455"}
{"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:19.815390914Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3478"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:19.816595353Z","hostname":"sock","pid":1590,"uri":"/cockroachdb","method":"POST","req_id":"208495d1-feba-401f-9136-3b8c8e5dc200","remote_addr":"[fd00:1122:3344:101::1]:47419","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":12890664,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:19.818096768Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58557"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:19.818727067Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"f3d7b26c-41f6-44e7-99a8-12ea86bb92b3","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58557","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":241,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:19.857147955Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:19.857291757Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:34328"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:19.857983018Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:19.857992478Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:19.882314739Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:19.88235988Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:19.902426132Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:19.902462733Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:19.921419028Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:19.921455428Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:19.939494448Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:19.939549259Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:19.957215853Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:19.957255053Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:19.974890196Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:19.974927397Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:19.992577371Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:19.992609291Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.009586494Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.009620114Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.070623235Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.070676766Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.128954952Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.128992663Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.205509762Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.205580373Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.285397135Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.285458066Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.359213621Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.359278502Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.437922686Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.437973387Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.512839979Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.51289712Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.587168613Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.587216754Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.654348973Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.654398053Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.714732063Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.714788304Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.777281918Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.777328999Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.836239055Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.836304176Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.83655713Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.836709163Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:20.934854759Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:20.934914Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:20.934922761Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:20.934929901Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:20.934936131Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:20.934943501Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:20.934950701Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:20.934973712Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:20.934981101Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:20.934987042Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:20.934993172Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:20.934998832Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:20.935006832Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:20.935013952Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:20.935021832Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:20.935027882Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:20.935033842Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:20.935040753Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:20.935046303Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:20.935174785Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"51d3f4c5-7ab1-40eb-8ddf-997e47ecba49","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized), (\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized)]","latency_us":1077816,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:20.935586111Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"145.300529ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"51d3f4c5-7ab1-40eb-8ddf-997e47ecba49\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:18:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"51d3f4c5-7ab1-40eb-8ddf-997e47ecba49\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.081843081Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.082210457Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.082218057Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.105705015Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.105743695Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.126342996Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.126379287Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.145817049Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.14585538Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.164726113Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.164769693Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.185015289Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.185051739Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.204244198Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.204283238Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.222925218Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.222957788Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.242669765Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.242704796Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.263304547Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.263342557Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.283122635Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.283164906Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.302247622Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.302280653Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.321305368Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.321339179Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.340436086Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.340467276Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.361977782Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.362016273Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.382811477Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.382847367Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.403305576Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.403341397Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.421164093Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.421201664Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.438079265Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.438127025Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.45461331Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.454654561Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.470660178Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.470707749Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.470970283Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.471329439Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:21.563319696Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:21.563379757Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:21.563388447Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:21.563397127Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:21.563403228Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:21.563410797Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:21.563416818Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:21.563434148Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:21.563441508Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:21.563447148Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:21.563452908Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:21.563458818Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:21.563466449Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:21.563472769Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:21.563479959Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:21.563483999Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:21.563489849Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:21.563496639Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:21.563502769Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.563627661Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"ebdbbc49-dd52-4224-9722-c49913ac180f","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized)]","latency_us":481629,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:21.563816844Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"136.402799ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ebdbbc49-dd52-4224-9722-c49913ac180f\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:18:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ebdbbc49-dd52-4224-9722-c49913ac180f\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.702241878Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.702587754Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.702595384Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.724363294Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.724407124Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.743657773Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.743703584Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.764108352Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.764154273Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.783631186Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.783693347Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.802434188Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.802477669Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.821628836Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.821672407Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.844012646Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.844055897Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.863951786Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.863990647Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.883103144Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.883145605Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.903064845Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.903104075Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.923428092Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.923468962Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.942468998Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.942517168Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.961680126Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.961721847Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.981446734Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.981485145Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.99990787Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:21.999958861Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.023033862Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.023083393Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.045055846Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.045101127Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.065692747Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.065749158Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.085706259Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.08574565Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.106204488Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.106262019Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.106459653Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.106598845Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.203353649Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.20340285Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.20341152Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.20341806Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.20342401Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.203430271Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.20343486Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.203459411Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.203465411Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.203471581Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.203476201Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.203480871Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.203486431Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.203492441Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.203496892Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.203502122Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.203507752Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.203513182Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.203518852Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.203659814Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"185fbbcd-1121-49c4-a039-16e240535e3d","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized), (\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized)]","latency_us":501272,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.203852557Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"105.651113ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"185fbbcd-1121-49c4-a039-16e240535e3d\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:18:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"185fbbcd-1121-49c4-a039-16e240535e3d\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.311234203Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.31172004Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.311730711Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.334184221Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.334233062Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.355464353Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.355508494Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.376095485Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.376143425Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.395970934Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.396021235Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.414916988Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.414966839Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.435364847Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.435413008Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.456051129Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.45609076Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.474422924Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.474460985Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.494228883Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.494273263Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.515652247Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.515693537Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.536290798Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.536328839Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.556415761Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.556451962Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.575574489Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.57561955Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.593720781Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.593759171Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.61358988Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.61363053Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.631908474Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.631953995Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.651432608Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.651470128Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.670752428Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.670807569Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.691101965Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.691138796Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.710204062Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.710248263Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.710474996Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.710605648Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.800430822Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.800480832Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.800488983Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.800496143Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.800502253Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.800507943Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.800514913Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.800536963Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.800543024Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.800550373Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.800556224Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.800563574Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.800569614Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.800577004Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.800582964Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.800588344Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.800595164Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.800601374Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.800611625Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:22.800756937Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"1ead8785-92ae-424e-b422-9886d0eea432","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized)]","latency_us":489233,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:22.80095643Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"226.609826ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1ead8785-92ae-424e-b422-9886d0eea432\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:18:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1ead8785-92ae-424e-b422-9886d0eea432\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.02940509Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.029720957Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.029731401Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.050965212Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.051009491Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.070696762Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.070737178Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.088077377Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.088118047Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.105994209Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.106041657Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.123662338Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.123711624Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.14244471Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.142488773Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.160530568Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.160571968Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.179280552Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.179322815Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.196998948Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.197038558Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.215737017Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.215773552Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.236258733Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.236292502Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.253734613Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.253767378Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.271112031Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.271156318Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.288235503Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.288273914Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.304256877Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.304309631Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.320215666Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.320260356Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.339605901Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.339645558Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.357672238Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.357717193Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.374570363Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.374605313Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.392615327Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.392663852Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.392837349Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.392961599Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:23.486104839Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:23.48615958Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:23.486167184Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:23.486175312Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:23.486181499Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:23.486188222Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:23.486193049Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:23.486222188Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:23.486230681Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:23.486239119Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:23.486313252Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:23.486319394Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:23.486326988Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:23.486334572Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:23.486342138Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:23.486348175Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:23.486355656Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:23.486360044Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:23.48636596Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.486448602Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"5915ec6c-7db1-48da-a2ca-a0084b924722","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized)]","latency_us":455653,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:23.486638861Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"130.53529ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5915ec6c-7db1-48da-a2ca-a0084b924722\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:18:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5915ec6c-7db1-48da-a2ca-a0084b924722\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.618359442Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.618709406Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.618718191Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.63891756Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.638960779Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.659014275Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.659049826Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.677302294Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.67734316Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.694664569Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.69470507Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.712733381Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.712772457Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.730862128Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.730966889Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.749317025Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.749354629Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.765594887Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.765634552Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.783039503Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.783138295Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.800938364Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.800970231Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.819852147Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.819889029Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.83894944Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.838989819Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.858203418Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.858244079Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.876966121Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.877006941Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.894853849Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.894902638Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.911087115Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.911119252Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.930916061Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.930948367Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.950096668Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.95014645Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.96868203Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.968721322Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.986788225Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.986836432Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.987032223Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:23.987146814Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.08027937Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.080334168Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.080342652Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.080348746Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.080355918Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.08036174Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.080377068Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.080382741Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.080389847Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.080393794Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.080399091Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.080404181Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.080409694Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.080415366Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.080420962Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.080427581Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.0804345Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.080440312Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.080445873Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.080567956Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"4216c7d7-908d-4647-893e-e7629de5e558","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized)]","latency_us":460830,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.080761384Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"335.164726ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4216c7d7-908d-4647-893e-e7629de5e558\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:18:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4216c7d7-908d-4647-893e-e7629de5e558\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.418239344Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.418602556Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.418611359Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.440732125Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.440779423Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.460492855Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.460539393Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.479199684Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.479239453Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.497112091Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.497160185Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.514560707Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.514610611Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.534458361Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.534503352Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.552013576Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.552052193Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.569954415Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.569994043Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.587931876Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.587973304Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.606945275Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.606989967Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.624641501Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.624680511Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.64262664Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.642667778Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.660796448Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.660836067Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.679152807Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.679194386Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.697326232Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.69737473Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.715744498Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.715785233Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.733916561Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.733959706Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.752542973Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.752588545Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.772074252Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.772118925Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.793140305Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.793204431Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.793381144Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.793495924Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.885259738Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.88530983Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.885319327Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.885327192Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.885333522Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.885341715Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.885368256Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.885376806Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.885384887Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.885391431Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.885397722Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.885402747Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.885410604Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.885418366Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.885425857Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.885433666Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.885438466Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.88544608Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.885452736Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:24.885584616Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"255d1344-f5ba-45b4-8a43-9eabef47567c","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized), (\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized)]","latency_us":465915,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:24.885736991Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"305.18995ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"255d1344-f5ba-45b4-8a43-9eabef47567c\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:18:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"255d1344-f5ba-45b4-8a43-9eabef47567c\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.193056685Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.193390806Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.193398231Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.215797748Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.215851712Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.236245073Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.236291649Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.255408521Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.255459156Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.273750386Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.273801715Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.295612204Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.295666382Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.315036019Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.315080692Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.334624248Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.334668095Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.35389176Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.353941898Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.373582095Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.373627376Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.392951059Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.392991832Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.411356875Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.411400816Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.429411427Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.429454469Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.448369406Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.448419291Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.466668954Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.466704674Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.485273814Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.4853255Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.503784131Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.503828428Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.522498213Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.522549129Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.541302024Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.541356492Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.559806113Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.559919712Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.578814431Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.578867542Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.579087805Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.579195844Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:25.670285701Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:25.670339712Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:25.670347399Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:25.670352809Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:25.670356718Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:25.670363112Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:25.670376135Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:25.670382613Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:25.670387713Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:25.670393094Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:25.67040441Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:25.670411394Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:25.670416813Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:25.670423206Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:25.670428392Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:25.670433706Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:25.670438836Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:25.670445304Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:25.670450217Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:25.670572881Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"fde217cc-85a9-4164-b2ca-a3157301bc7c","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized), (\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized)]","latency_us":476066,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:25.670744109Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"347.410352ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fde217cc-85a9-4164-b2ca-a3157301bc7c\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:18:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fde217cc-85a9-4164-b2ca-a3157301bc7c\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.020203149Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.020514047Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.020521397Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.042750939Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.042794627Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.062614903Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.062654119Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.081098545Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.081140358Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.101278166Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.101393567Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.119309207Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.119438138Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.137149128Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.137183188Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.15471595Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.154756807Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.172960423Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.17301118Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.191170956Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.19122251Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.21122339Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.211266703Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.230402374Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.230447572Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.249455557Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.249503014Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.268854445Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.268906599Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.288599633Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.28863599Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.308127213Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.308177539Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.327849981Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.327901489Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.347171574Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.347210527Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.367245388Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.367291664Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.386595271Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.386631637Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.406200568Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.406246816Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.406424672Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.406560922Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:26.498454584Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:26.498514782Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:26.498524607Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:26.498531272Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:26.498537554Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:26.498544894Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:26.498571033Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:26.498577258Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:26.498584842Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:26.498590795Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:26.498598577Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:26.498606386Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:26.498612462Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:26.498616483Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:26.498622268Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:26.498626721Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:26.498630949Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:26.498635149Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:26.498640868Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.498712944Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"9337362f-87da-471a-93f7-369ed4366f5e","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized)]","latency_us":477096,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:26.498893884Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"346.748555ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9337362f-87da-471a-93f7-369ed4366f5e\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:18:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9337362f-87da-471a-93f7-369ed4366f5e\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.848279003Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.848626926Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.848641224Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.871047409Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.871093694Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.892684389Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.89273644Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.913038817Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.913091562Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.931706534Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.931754422Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.950461534Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.950513482Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.968997341Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.969046692Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.987562277Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:26.987606912Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.00633168Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.006374861Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.026569172Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.026618054Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.047184702Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.047225719Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.06684268Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.06688155Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.0855061Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.085546179Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.104141631Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.104196016Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.12668621Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.126727564Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.14749885Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.147553854Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.16768263Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.167728418Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.187652575Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.187691491Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.207076879Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.207136861Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.22591684Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.225955297Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.244113233Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.244165405Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.24433818Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.244522519Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:27.339509579Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:27.33956284Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:27.339569149Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:27.339573247Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:27.339578422Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:27.339585369Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:27.339598625Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:27.339603978Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:27.339609041Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:27.339614384Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:27.339620694Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:27.339625756Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:27.339630369Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:27.339635413Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:27.33964056Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:27.339645566Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:27.339716461Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:27.339723202Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:27.339728274Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:27.339821257Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"fc930a9d-5520-4565-a80a-95220bf99979","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized)]","latency_us":490049,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:27.340004644Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"672.616987ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fc930a9d-5520-4565-a80a-95220bf99979\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:18:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fc930a9d-5520-4565-a80a-95220bf99979\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.015202776Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.01644264Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.016454537Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.039861814Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.039910218Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.066107734Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.066158688Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.090674102Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.090792249Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.113715249Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.113852294Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.136714716Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.13676822Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.158848276Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.158967682Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.182239131Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.182288303Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.204368107Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.204420955Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.223243857Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.223283289Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.24364093Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.243694565Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.26434746Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.264392077Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.281309479Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.281354967Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.301366618Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.301428306Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.319082304Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.319131355Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.338483013Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.338526767Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.355972262Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.356011768Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.373015676Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.37305853Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.392856464Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.392906058Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.413013062Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.413055897Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.431849341Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.43189856Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.432096216Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.43220464Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:28.520952011Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:28.521012472Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:28.521021415Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:28.5210296Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:28.52103754Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:28.521043512Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:28.521067176Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:28.521075098Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:28.52108421Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:28.521092348Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:28.52109846Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:28.521106101Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:28.521112279Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:28.521119948Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:28.521124355Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:28.521130327Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:28.521134573Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:28.521138661Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:28.521142992Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:28.521216896Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"377de076-8449-4102-9d19-0d0862f4c990","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized)]","latency_us":504414,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:28.521415013Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"786.419736ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"377de076-8449-4102-9d19-0d0862f4c990\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:18:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"377de076-8449-4102-9d19-0d0862f4c990\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.311150783Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.311450357Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.311462873Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.333364821Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.333413009Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.354419172Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.354465579Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.373958954Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.374009148Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.392420464Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.392471849Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.41170023Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.411758262Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.430263893Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.430373535Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.449990747Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.450034829Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.468999148Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.469040455Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.489141876Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.48919298Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.507741305Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.507784618Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.526654014Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.526697036Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.54475114Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.544792287Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.562848524Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.562897978Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.581788087Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.581826778Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.60088324Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.600930772Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.618265895Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.618313633Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.637385432Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.637432963Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.655133982Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.655185817Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.674478606Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.674522782Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.693265474Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.693318228Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.6935059Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.693629633Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:29.782062745Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:29.782111158Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:29.782117402Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:29.782122642Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:29.782128014Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:29.78213333Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:29.782147581Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:29.78215435Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:29.78215929Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:29.7821656Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:29.78217069Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:29.782177365Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:29.782183984Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:29.782189393Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:29.782194484Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:29.782200738Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:29.782205829Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:29.782212176Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:29.782215813Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:29.782363804Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"a4229168-89b1-40b2-bb1f-c009761f8065","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized)]","latency_us":469809,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:29.782582461Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"753.283715ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a4229168-89b1-40b2-bb1f-c009761f8065\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:18:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a4229168-89b1-40b2-bb1f-c009761f8065\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.539625824Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.540000174Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.540012643Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.563305343Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.56335129Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.585079214Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.585123473Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.605972976Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.606011039Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.627789987Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.627836966Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.646129568Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.646166796Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.664307202Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.664340878Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.683393569Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.683433544Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.704275399Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.704312572Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.724352119Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.724401883Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.742519763Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.74255747Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.761812534Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.761860863Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.779749301Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.77979522Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.79916769Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.79922485Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.816830535Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.816875451Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.835571301Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.83561544Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.854832216Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.854876034Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.872765913Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.872807896Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.891468007Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.891516279Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.91022794Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.910278097Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.929040919Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.929095473Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.929316914Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:30.929520542Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:31.027011052Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:31.027062616Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:31.027070575Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:31.027076191Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:31.027081684Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:31.027087451Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:31.027110213Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:31.027115923Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:31.027121126Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:31.027127876Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:31.027132789Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:31.027139502Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:31.027144864Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:31.027149983Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:31.027155992Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:31.027160896Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:31.02716608Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:31.027172717Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:31.027178042Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.027259129Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"3933c42b-55ea-4d4e-9ec8-62d75788432c","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized), (\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized)]","latency_us":486140,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:31.027456438Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"563.328159ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3933c42b-55ea-4d4e-9ec8-62d75788432c\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:18:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3933c42b-55ea-4d4e-9ec8-62d75788432c\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.593346923Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.593656312Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.593666598Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.616440608Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.616490277Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.637234837Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.637348245Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.656573868Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.656624569Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.676830638Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.676870867Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.698234425Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.698275619Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.718128488Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.718181261Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.736963902Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.737007121Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.755857471Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.755904853Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.7745955Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.774705787Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.794901405Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.794941971Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.814527809Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.814582354Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.832637416Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.832669469Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.852325307Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.852372427Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.871452114Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.871499187Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.891305533Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.891342827Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.91089323Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.910939899Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.930153328Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.930197204Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.949019136Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.949055099Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.968514378Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.968558478Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.988096671Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.988145909Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.988422243Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:31.988572245Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:32.084455982Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:32.084589824Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:32.084600239Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:32.084606858Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:32.084614528Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:32.084622056Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:32.084638818Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:32.084646956Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:32.08465469Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:32.08466249Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:32.084668726Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:32.084676357Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:32.084684138Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:32.084691947Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:32.084699597Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:32.084705888Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:32.084713266Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:32.084719022Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:32.084726654Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:32.084810281Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"f2cc0239-77e1-492c-a5e9-2148e0b97aa2","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized), (\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized)]","latency_us":490011,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:32.085027025Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.087897627s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f2cc0239-77e1-492c-a5e9-2148e0b97aa2\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:18:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f2cc0239-77e1-492c-a5e9-2148e0b97aa2\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.177627506Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.177950639Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.177957464Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.202022416Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.202065355Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.223517633Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.223623834Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.244152251Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.244189621Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.262183748Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.262221211Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.281871994Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.281905154Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.301698334Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.301738516Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.323659232Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.32370383Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.343171041Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.343210698Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.362235883Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.362277349Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.381121515Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.381160112Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.400611379Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.400646827Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.422899735Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.422936608Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.441741248Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.441791171Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.460780204Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.460811273Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.480619204Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.480655822Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.499789942Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.499825277Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.519546257Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.519589945Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.538059433Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.538093512Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.557324487Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.557355294Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.576031865Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.576083494Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.576336097Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.576477016Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:33.67287972Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:33.672927768Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:33.672935831Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:33.672941362Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:33.672948749Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:33.67296901Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:33.67297516Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:33.672980447Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:33.672986775Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:33.672992213Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:33.672997407Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:33.673002432Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:33.673007579Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:33.673012923Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:33.67301807Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:33.673024276Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:33.673029761Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:33.673034973Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:33.67304012Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:33.673188285Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"2a084cf8-4434-4330-9a6c-a4142083f794","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized)]","latency_us":494094,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:33.673399638Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.349008433s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2a084cf8-4434-4330-9a6c-a4142083f794\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:18:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2a084cf8-4434-4330-9a6c-a4142083f794\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.027605055Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.027942354Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.027950051Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.049021608Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.049058105Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.067904348Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.067942467Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.084441519Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.084481756Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.101441726Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.101482012Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.117962652Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.117993609Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.134851709Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.134890774Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.152846883Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.152883351Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.172371349Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.172407219Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.189568318Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.189607187Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.205841124Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.205869642Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.223800151Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.223834165Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.241930335Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.241966795Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.259691536Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.259735956Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.278079196Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.278182381Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.298333857Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.298370242Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.31878267Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.318830081Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.337982139Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.338019105Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.357168418Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.357205506Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.375768217Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.375800505Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.393737789Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.393781392Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.39400799Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.394199919Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:35.487440298Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:35.487495649Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:35.48750389Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:35.487512674Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:35.487518918Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:35.487534677Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:35.487541362Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:35.487547868Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:35.487554488Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:35.487562831Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:35.487568203Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:35.487572919Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:35.487577747Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:35.487582594Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:35.487587431Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:35.48759225Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:35.487597192Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:35.4876038Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:35.487609295Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:35.487783677Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"49e1a94a-5afb-4b08-bf95-e1bb3f4862c6","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized), (\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized)]","latency_us":458791,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:35.488003375Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"2.233787614s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"49e1a94a-5afb-4b08-bf95-e1bb3f4862c6\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:18:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"49e1a94a-5afb-4b08-bf95-e1bb3f4862c6\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.728989178Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.729351903Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.729361831Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.751726239Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.75186725Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.768792091Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.768917015Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.7838063Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.78384996Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.801428152Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.801477222Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.818619547Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.818666516Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.836195707Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.836238982Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.852908086Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.852957728Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.870949096Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.870994284Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.890980788Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.89101917Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.910855623Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.910898833Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.929166382Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.929209377Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.947526947Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.947575687Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.965067156Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.965125526Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.98332839Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:37.98337265Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:38.000396365Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:38.000508592Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:38.019424192Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:38.019477555Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:38.038898559Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:38.038956356Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:38.060772008Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:38.060835075Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:38.082487826Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:38.082535724Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:38.102080324Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:38.102140025Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:38.102309996Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:38.102426211Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:38.195596661Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:38.195646471Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:38.195654796Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:38.195660552Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:38.195667002Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:38.195689137Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:38.195696337Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:38.195701662Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:38.195706969Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:38.195712294Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:38.195716287Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:38.195723178Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:38.195728822Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:38.195732666Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:38.195738028Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:38.195743372Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:38.195748453Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:38.19575377Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:38.195762432Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:38.19592766Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"d6a7c06d-ad65-4c03-be8a-00d944175c43","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized)]","latency_us":465575,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:38.196218016Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.051034856s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d6a7c06d-ad65-4c03-be8a-00d944175c43\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:18:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d6a7c06d-ad65-4c03-be8a-00d944175c43\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.251428555Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.251730284Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.251739651Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.271982428Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.272029782Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.290304726Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.290352184Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.310576304Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.310628271Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.329237912Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.329282537Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.346810178Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.346852798Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.365020635Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.365063039Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.382817995Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.382885009Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.40131465Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.401376058Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.420681222Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.42072462Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.441332802Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.441446605Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.460986629Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.461038211Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.480262393Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.480310206Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.49902704Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.499074666Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.517002519Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.517052263Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.536337707Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.536387151Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.555158919Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.555201436Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.574279643Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.574327559Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.594053509Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.5940943Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.61366394Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.61370955Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.633537423Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.633600003Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.633968493Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.634453217Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:39.731739689Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:39.731785965Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:39.7317922Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:39.731797712Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:39.731802821Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:39.731817671Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:39.731822912Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:39.731828987Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:39.731833806Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:39.731839975Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:39.731845094Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:39.731850129Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:39.731855829Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:39.731862579Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:39.731867444Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:39.731872357Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:39.731878357Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:39.731883644Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:39.731889963Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:39.731974021Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"4454c798-78fb-4b00-ab90-fa38acc8e537","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized), (\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized)]","latency_us":479135,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:39.732233721Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.628117404s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4454c798-78fb-4b00-ab90-fa38acc8e537\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:18:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4454c798-78fb-4b00-ab90-fa38acc8e537\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.365618529Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.366045688Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.366054182Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.387536835Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.387656973Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.408304102Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.408342531Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.426654488Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.426764868Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.445847466Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.445907973Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.464250763Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.464291498Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.483830096Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.483936478Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.502694958Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.502738403Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.521331607Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.521376618Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.540669602Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.540712419Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.559460334Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.559494543Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.579097501Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.579205644Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.599684248Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.599725377Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.619027965Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.619099731Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.640056161Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.640097084Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.658767098Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.658815661Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.676968026Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.677008199Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.696222172Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.696269667Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.716216967Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.71626673Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.735757213Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.735797405Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.754963653Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.755014936Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.755225651Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.755360663Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:41.846310646Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:41.846359583Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:41.846366128Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:41.846371603Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:41.846376891Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:41.846398181Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:41.846404866Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:41.84641035Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:41.846415488Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:41.846420475Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:41.846425473Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:41.8464306Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:41.846435429Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:41.846439985Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:41.846444954Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:41.84644981Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:41.846455932Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:41.846459485Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:41.846462813Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:41.846548464Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"56e805fb-5424-40e3-9c1b-e4593baf4769","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized)]","latency_us":479463,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:41.846753996Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"3.100948487s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"56e805fb-5424-40e3-9c1b-e4593baf4769\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:18:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"56e805fb-5424-40e3-9c1b-e4593baf4769\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:44.956720444Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:44.956984588Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:44.956992613Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:44.981659108Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:44.981710822Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.002094029Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.002145536Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.025117856Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.025170629Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.045980775Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.046026995Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.065481907Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.065592987Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.086439056Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.086483897Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.106734985Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.106785282Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.126863579Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.126928108Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.146247957Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.146295629Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.167109754Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.167155242Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.189521479Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.189566499Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.210033159Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.210078872Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.229918629Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.229966929Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.250604491Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.250646519Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.270879882Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.270921283Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.29341669Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.293457416Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.314175415Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.314292583Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.335551049Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.335595974Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.356239031Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.356290932Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.377843109Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.377909719Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.378102885Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.378239387Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:45.481015281Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:45.481130998Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:45.481143252Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:45.48115184Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:45.481175202Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:45.481192893Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:45.481197899Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:45.481203984Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:45.481208756Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:45.481213369Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:45.481218159Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:45.481224356Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:45.481234059Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:45.481239431Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:45.481251769Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:45.481258613Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:45.481265138Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:45.481270942Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:45.481275845Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:45.481286982Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"9b333612-573f-4e9c-9d35-c63bcfe85d06","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized), (\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized)]","latency_us":522989,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:45.481475835Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"2.960571695s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9b333612-573f-4e9c-9d35-c63bcfe85d06\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:18:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9b333612-573f-4e9c-9d35-c63bcfe85d06\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.451607348Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.452017593Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.452025984Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.475187303Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.475241904Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.495554152Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.495613572Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.514449568Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.514500981Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.53659297Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.536646821Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.555787464Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.555845281Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.575383909Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.575440375Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.593688674Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.593740331Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.614570817Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.614641065Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.635425643Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.635476924Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.655730173Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.655789798Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.675919959Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.675974794Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.697047825Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.697093248Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.716497213Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.716556482Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.735843968Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.735905666Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.75436007Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.754405671Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.774540288Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.774654215Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.795248035Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.795298989Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.815558643Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.815603457Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.834581126Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.834629839Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.854498265Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.854563666Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.854723015Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.854909431Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:48.948969444Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:48.949110306Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:48.949118838Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:48.949132085Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:48.949136294Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:48.949157763Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:48.949162038Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:48.949165957Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:48.949171366Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:48.949176729Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:48.949182279Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:48.949187651Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:48.94919621Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:48.949200729Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:48.949205998Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:48.949210198Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:48.949215551Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:48.949223285Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:48.949227158Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:48.949231854Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"f2f8e5a2-511b-45f2-9685-e35ec0e01cc7","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized)]","latency_us":496082,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:48.949456483Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"2.91721042s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f2f8e5a2-511b-45f2-9685-e35ec0e01cc7\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:18:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f2f8e5a2-511b-45f2-9685-e35ec0e01cc7\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:49.931170015Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56904"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:49.931251607Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"e1d0a888-2186-4a90-8fa9-3c9b30351717","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56904","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":58,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:51.876014066Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:51.87710476Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:51.877114707Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:51.90054564Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:51.900603513Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:51.9214749Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:51.921525638Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:51.943273977Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:51.943329299Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:51.963366969Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:51.963415513Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:51.983385238Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:51.983436267Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.002033819Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.00209397Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.022951756Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.023013669Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.04344955Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.043500091Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.064273488Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.06432762Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.085358041Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.085409903Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.105639454Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.105690053Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.125971992Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.126037825Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.146400335Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.146460138Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.165166551Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.165212883Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.183921564Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.183975031Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.205772029Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.205885587Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.225781841Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.225827534Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.245676504Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.245727834Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.265638318Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.265682953Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.287378515Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.287436894Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.28761836Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.287741362Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:52.381031725Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:52.381104767Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:52.381112961Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:52.381120696Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:52.381131571Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:52.381137787Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:52.381144218Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:52.381149693Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:52.381155365Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:52.38116083Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:52.381167543Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:52.381173187Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:52.381179759Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:52.381185197Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:52.381191778Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:52.381195828Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:52.381199428Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:52.381213819Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:52.381223381Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:52.381283495Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"b1dcaefa-2cc3-4195-858e-43ec412821c0","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized)]","latency_us":503813,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:52.381572557Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"3.160904927s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b1dcaefa-2cc3-4195-858e-43ec412821c0\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:18:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b1dcaefa-2cc3-4195-858e-43ec412821c0\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.551447054Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.551777182Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.551784738Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.573184027Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.573250497Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.592934929Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.593059731Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.612875478Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.612928943Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.631197483Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.631257034Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.649947166Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.649997407Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.668014641Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.668071276Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.686032712Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.686107376Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.705432636Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.705488137Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.723390397Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.723450407Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.741333474Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.743265635Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.760793766Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.760845179Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.779824337Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.779873565Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.801397014Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.80145004Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.820166694Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.820217161Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.839142055Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.839197509Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.85682893Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.856894696Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.876200761Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.876247824Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.892346212Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.892390144Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.908991971Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.9090436Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.927138572Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.927213179Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.927392272Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:55.92751159Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:56.015820698Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:56.015898503Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:56.015907813Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:56.015916569Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:56.015924557Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:56.015932563Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:56.01594056Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:56.015946888Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:56.015953657Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:56.015962451Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:56.015968761Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:56.015977123Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:56.015985026Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:56.015989901Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:56.015995976Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:56.016009814Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:56.016014718Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:56.016024524Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:56.016039008Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:18:56.016090112Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"80a457b0-e6a8-4e67-a617-7f93f7ba9c7b","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized)]","latency_us":463274,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:18:56.016372265Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"4.736445531s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"80a457b0-e6a8-4e67-a617-7f93f7ba9c7b\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:18:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"80a457b0-e6a8-4e67-a617-7f93f7ba9c7b\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.766599893Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.76741451Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.767426089Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.790563581Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.790619317Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.807517453Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.80756551Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.827474689Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.827522474Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.845565931Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.845610876Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.864057467Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.864108702Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.881912057Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.884152326Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.89906548Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.899117071Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.916767466Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.916823042Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.935831674Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.935879806Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.954414222Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.954530027Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.973022191Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.973076539Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.991197456Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:00.991244632Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:01.009727022Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:01.009784144Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:01.027499127Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:01.027551843Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:01.04535365Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:01.04539805Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:01.06313135Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:01.06318176Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:01.081264009Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:01.08133721Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:01.099320994Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:01.099369867Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:01.11797787Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:01.118024802Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:01.136181409Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:01.136249444Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:01.136494192Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:01.136637172Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:01.226512246Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:01.226584134Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:01.226594166Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:01.226605248Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:01.22661061Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:01.226617904Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:01.226625048Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:01.226632332Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:01.226643282Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:01.226651757Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:01.226659567Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:01.226667732Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:01.226676142Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:01.226683296Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:01.226691077Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:01.226698802Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:01.226707849Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:01.226726225Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:01.226732553Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:01.226810947Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"f75b0a73-670b-485f-a020-24e098190723","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized)]","latency_us":458839,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:01.227028114Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"6.040858915s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f75b0a73-670b-485f-a020-24e098190723\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:19:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f75b0a73-670b-485f-a020-24e098190723\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.285369627Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.286072781Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.286094015Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.308405395Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.310837829Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.326445305Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.328747738Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.345323433Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.345390606Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.363269852Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.363320881Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.382046987Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.382089127Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.399730519Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.399766575Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.417419354Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.41746683Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.435133729Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.435175289Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.452537097Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.452586204Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.470604588Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.470644705Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.489836089Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.489872671Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.507571134Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.507622445Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.526029792Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.526084064Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.543456861Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.54351374Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.561659741Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.561706055Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.580734904Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.580782277Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.598990695Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.599042521Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.617324062Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.617380397Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.6341627Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.634210457Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.652156004Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.652211796Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.652397123Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.652516666Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:07.74144397Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:07.741529191Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:07.741538857Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:07.741546469Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:07.741551203Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:07.741558525Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:07.741570685Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:07.74157662Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:07.741583942Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:07.741589876Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:07.741597086Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:07.741607089Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:07.741614364Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:07.741618779Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:07.741625615Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:07.741631248Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:07.741644158Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:07.741654677Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:07.741660218Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:07.741760345Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"5ce90cec-6367-4216-a5dd-93624cc1ce4f","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized)]","latency_us":454837,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:07.742048062Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"6.130627563s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5ce90cec-6367-4216-a5dd-93624cc1ce4f\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:19:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5ce90cec-6367-4216-a5dd-93624cc1ce4f\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:13.890766171Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:13.891670304Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:13.891697258Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:13.912049794Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:13.912128227Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:13.93143566Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:13.931485255Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:13.949748704Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:13.949799067Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:13.967250471Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:13.967324487Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:13.984634727Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:13.984754302Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.002757004Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.005059063Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.020699849Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.020831309Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.038392821Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.038449185Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.056683799Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.056739019Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.074996722Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.075059198Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.093412728Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.093457888Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.111414421Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.111470634Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.12973554Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.129801879Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.147745163Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.147792835Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.165857408Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.165911793Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.183973021Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.18403108Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.202076667Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.202135431Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.219865239Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.219918424Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.238100557Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.238159311Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.255997974Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.256069732Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.256332649Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.256530621Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:14.349940028Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:14.350019764Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:14.350029205Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:14.350039096Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:14.350045752Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:14.350054818Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:14.350062833Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:14.350067906Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:14.350081772Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:14.350090425Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:14.350109334Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:14.350116187Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:14.350124578Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:14.350132632Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:14.350141032Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:14.350149498Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:14.350169588Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:14.350184598Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:14.350191405Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:14.350290753Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"b7568256-3b9f-453a-9dcc-e8f060d4d517","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized)]","latency_us":457872,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:14.350519675Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"6.199405038s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b7568256-3b9f-453a-9dcc-e8f060d4d517\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:19:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b7568256-3b9f-453a-9dcc-e8f060d4d517\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.036007745Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47741"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.036074566Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"2852c127-72c8-4184-ae94-32c75c56cfb3","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47741","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.559040898Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.559424055Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.559435595Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.582290622Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.582369053Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.606004373Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.606082754Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.628974512Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.629035893Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.650932535Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.650992766Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.674241639Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.67430743Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.697104356Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.697177617Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.71848088Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.718537621Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.740015196Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.740079317Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.763819258Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.763881089Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.786726856Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.786784667Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.808999064Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.809058575Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.831980063Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.832036434Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.855081534Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.855145095Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.877030547Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.877090808Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.898872118Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.898928949Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.920471695Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.920531966Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.94444449Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.944511401Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.965938795Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.965997456Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.989210869Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:20.98927141Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:21.012458893Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:21.012535734Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:21.012781058Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:21.012944251Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:21.118187152Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:21.118269703Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:21.118278723Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:21.118287773Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:21.118295343Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:21.118301623Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:21.118308964Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:21.118316394Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:21.118322284Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:21.118328644Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:21.118336074Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:21.118342164Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:21.118349434Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:21.118367295Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:21.118375335Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:21.118385965Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:21.118390585Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:21.118396425Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:21.118400945Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:21.118479586Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"769d1a7c-b9ee-4649-9f90-eee9401b5eb6","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\", TimeNotSynchronized), (\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\", TimeNotSynchronized), (\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\", TimeNotSynchronized), (\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\", TimeNotSynchronized), (\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\", TimeNotSynchronized), (\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\", TimeNotSynchronized), (\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\", TimeNotSynchronized), (\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\", TimeNotSynchronized), (\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\", TimeNotSynchronized), (\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\", TimeNotSynchronized), (\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\", TimeNotSynchronized), (\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\", TimeNotSynchronized), (\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\", TimeNotSynchronized), (\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\", TimeNotSynchronized), (\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\", TimeNotSynchronized), (\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\", TimeNotSynchronized), (\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\", TimeNotSynchronized), (\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\", TimeNotSynchronized), (\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\", TimeNotSynchronized)]","latency_us":559260,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:21.11872259Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"15.918762729s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"769d1a7c-b9ee-4649-9f90-eee9401b5eb6\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:19:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"769d1a7c-b9ee-4649-9f90-eee9401b5eb6\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: dcbedba4-6a83-41c6-9a2a-514fb195d796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0755d222-4cb0-4d69-ba56-feb7502debc0, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6d20efb2-6aac-47eb-adde-bd47312ac60a, kind: Service { id: 0755d222-4cb0-4d69-ba56-feb7502debc0 }, name: Name(\"ntp-0755d222-4cb0-4d69-ba56-feb7502debc0\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 220])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5e3ff898-c77b-42a1-9764-187791149401, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac169011-c605-4b75-aa7b-c3144c1ddd86, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: e9e24743-a540-4b93-9133-4eae210ed8f3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a1c22271-87f5-44b0-b506-06dff6539283, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d696101-87c5-4c96-b623-2e415a336b1d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ea63e896-0dc6-4da7-b3d5-27de7463f89a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4dfbb131-ca76-448d-8328-3ae1e5fa72b9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 82e8a1b2-fbe0-4447-944a-27decf04f25a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 652e3013-976a-43ba-bb68-b6f497ea4197, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4bd02937-0d24-4783-b773-973d42c4eef8, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External } } } }, OmicronZoneConfig { id: a64df5e3-8d68-4581-8828-def85de45a96, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External } } } }, OmicronZoneConfig { id: c8b70fc0-b959-4973-889c-bcdff32c973f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External } } } }, OmicronZoneConfig { id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External } } } }, OmicronZoneConfig { id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External } } } }, OmicronZoneConfig { id: 5145b222-0ffb-403b-a919-074d655b57e3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 43c9638c-6930-4233-a47d-c58f94e587c5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.021365017Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.022864161Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.022872671Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dcbedba4-6a83-41c6-9a2a-514fb195d796, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.044490179Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.04455958Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed9071c1-21f2-4f15-812c-6b6ce9f54d7b, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.06262968Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.062675841Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2459ecd0-1d68-43a3-8a7a-97b992a6728f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.080566798Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.080638769Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e3ff898-c77b-42a1-9764-187791149401, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.097737274Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.097792465Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ac169011-c605-4b75-aa7b-c3144c1ddd86, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.114694267Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.114745797Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8cc9856d-c4e4-49e3-9237-1d97f538eae8, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.135060074Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.135126425Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e9e24743-a540-4b93-9133-4eae210ed8f3, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.153357588Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.153412299Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6a642efd-fc7d-4434-bb44-b7f02c3b3137, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.170467093Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.170525634Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 41ef6d1d-d04a-4944-8844-640cc5b2f887, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.188610744Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.188669035Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 350afff8-3a4b-48f9-b954-9cd28e0a2019, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.20635464Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.208569455Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ff8af3cb-0e96-47e9-b05c-afa245f1dff5, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.224500221Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.224561552Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4bd02937-0d24-4783-b773-973d42c4eef8, dataset_name: DatasetName { pool_name: ZpoolName { id: d264dd09-dd1e-4b44-ac20-70d4e8728824 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.240981866Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.241039717Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a64df5e3-8d68-4581-8828-def85de45a96, dataset_name: DatasetName { pool_name: ZpoolName { id: 1962e339-1294-4041-a983-29e64bdb2ffe (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.258612689Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.258683701Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8b70fc0-b959-4973-889c-bcdff32c973f, dataset_name: DatasetName { pool_name: ZpoolName { id: 29f0e314-9945-472b-979d-b91f7fcb5170 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.275761845Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.275817866Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc, dataset_name: DatasetName { pool_name: ZpoolName { id: b4f1d1aa-0185-4f19-9f65-a972831259c3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.293201485Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.293262346Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14af2360-4bb4-4f0c-88c4-c36ebd6f02c7, dataset_name: DatasetName { pool_name: ZpoolName { id: fa79b281-89d1-42f4-b871-62a5aaaf9922 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.310807498Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.310857399Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d78f8b7a-b7a9-41c5-83d8-2b2ed0991813, dataset_name: DatasetName { pool_name: ZpoolName { id: 42895196-0869-4ea3-9c4e-285bbcea1a80 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.328600834Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.328665485Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ff6eb74-346e-45ea-890b-3a8a4237dcff, dataset_name: DatasetName { pool_name: ZpoolName { id: 0de19191-ee35-4d6e-ab04-5e684dafd16c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.346578693Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.346643314Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5145b222-0ffb-403b-a919-074d655b57e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.364350128Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.364400529Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 43c9638c-6930-4233-a47d-c58f94e587c5, dataset_name: DatasetName { pool_name: ZpoolName { id: 73bca2cf-2bf8-443a-bf93-681a0fe6b326 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.381753568Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.381821569Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.382105974Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.382259966Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.502898955Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.50320534Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.50322514Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.503257121Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.503284321Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.519430791Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.519700465Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.519716595Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.519730425Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.519754236Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.696553846Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.696949303Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.696990153Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.698352965Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.731700671Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.731962575Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.732045927Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.75531432Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.755369831Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.75589346Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.760956851Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.765927711Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.771183685Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.77707123Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.782287714Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.787458217Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.791869518Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.908781416Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.913173827Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.917637749Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.921991559Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.927137102Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.931899458Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.936711415Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.941464092Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Configuring new Omicron zone: oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.946516733Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.951395721Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.955946945Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.96065932Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.96686399Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.97310241Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.979014695Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.985450919Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.992613854Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:37.998548389Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.004293831Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.010242317Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.053470462Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.097321846Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.133709981Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.167726178Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.201035283Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.203233978Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.203251359Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.255357186Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 224, 67]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte1\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.255426267Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.257764795Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.257791155Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.312003866Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 146, 0]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte2\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.312077428Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.31475057Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.314790771Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.370611398Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 132, 246]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte3\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.370683259Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.372562559Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.37257896Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.423083501Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 128, 143]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte4\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.423149372Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.424945941Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.424965162Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.475056027Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 175, 152]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte5\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.475111297Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.2.167,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.477607837Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.2.167,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.477626328Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.2.164,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.47842355Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.2.164,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.478444661Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.2.168,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.478602483Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.2.168,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.478613233Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.2.166,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.478863288Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.2.166,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.478874368Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.2.165,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.47904946Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.2.165,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.479063871Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.2.164,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.48275125Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.484568189Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.2.166,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.48460788Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.2.165,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.486983178Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.487007439Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.2.168,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.489324136Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.489338406Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.491945138Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.2.167,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.515658819Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"Configuring new Omicron zone: oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.585011303Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.589986233Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.619137412Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.624393366Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.629579259Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.634796103Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Configuring new Omicron zone: oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.639634071Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.64638887Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.653073857Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.659765425Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:38.66445258Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:51.692305716Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64961"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:51.931947527Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"323cf4b4-817c-4a42-b2c8-dc9224b39628","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64961","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":98461,"response_code":"200"}
{"msg":"Profile for oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService14\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::11\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::11\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"17000\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:52.748209993Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService16\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::f\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::f\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"17000\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:52.819998607Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:52.820083368Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:53.195974179Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService17\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::10\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::10\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"17000\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:53.204251521Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService26\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::9\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte3\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.1.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.1.6\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/external_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:101::9]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"172.30.1.6:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:53.303049029Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService24\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::8\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte5\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.1.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.1.5\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/external_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:101::8]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"172.30.1.5:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:53.303324364Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:53.303388735Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:53.306315852Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:53.312625713Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService18\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::18\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_0de19191-ee35-4d6e-ab04-5e684dafd16c/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::18\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"552f5469-4328-4c4b-a032-97c58aeb483c\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:53.580764482Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService20\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::15\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_b4f1d1aa-0185-4f19-9f65-a972831259c3/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::15\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"53cb8c05-1a99-400b-924b-aec15855e2d9\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:53.582147434Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:53.582232806Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:53.588567637Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService21\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::17\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_42895196-0869-4ea3-9c4e-285bbcea1a80/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::17\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"6d09ccc9-f875-4081-8ea6-a6ca1129e047\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:53.595255065Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService22\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::19\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::19\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"ef2944a3-7118-477a-97b7-e7ea4983316d\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:53.595728823Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService23\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::1a\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_73bca2cf-2bf8-443a-bf93-681a0fe6b326/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::1a\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"72e998e9-d1f1-4859-b259-35ed1f28ceb1\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:53.596722928Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService11\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::16\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_fa79b281-89d1-42f4-b871-62a5aaaf9922/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::16\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"25d87d96-3e25-4020-8924-83cf421e2540\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:53.597787636Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService12\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::14\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_29f0e314-9945-472b-979d-b91f7fcb5170/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::14\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"559ebbd7-9759-463a-ac04-528bba0bf9c9\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:53.598289784Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService15\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::13\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_1962e339-1294-4041-a983-29e64bdb2ffe/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::13\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"af1fd904-257f-49c0-af11-9bcfebcf7d8b\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:53.599018865Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:53.599159287Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:53.605604411Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:53.611987274Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:53.618629971Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:53.624978422Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:53.630197596Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService13\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::d\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/oximeter\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"id\" value=\"ea63e896-0dc6-4da7-b3d5-27de7463f89a\"/>\n        <propval type=\"astring\" name=\"address\" value=\"[fd00:1122:3344:101::d]:12223\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:54.140652349Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:54.191642018Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\") failed: Property not found. retry in 73.335117ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.465825633Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\") failed: Property not found. retry in 67.789263ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.475471108Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\") failed: Property not found. retry in 41.811961ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.483540888Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\") failed: Property not found. retry in 41.158201ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.491719159Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\") failed: Property not found. retry in 25.022799ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.499958062Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\") failed: Property not found. retry in 72.326673ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.509291702Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\") failed: Property not found. retry in 57.958412ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.520412691Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\") failed: Property not found. retry in 30.967406ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.539875213Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\") failed: Property not found. retry in 47.072735ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.551235356Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\") failed: Property not found. retry in 32.239404ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.565903102Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\") failed: Property not found. retry in 66.588517ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.582502348Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\") failed: Property not found. retry in 66.143753ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.599577553Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\") failed: Property not found. retry in 73.083054ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.620820114Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\") failed: Property not found. retry in 102.385265ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.70587859Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\") failed: Property not found. retry in 106.412446ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.71955404Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\") failed: Property not found. retry in 84.358355ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.732186423Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\") failed: Property not found. retry in 64.575275ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.748507465Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\") failed: Property not found. retry in 127.222874ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.762735994Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\") failed: Property not found. retry in 54.11655ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.777119205Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\") failed: Property not found. retry in 139.909011ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.789208249Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\") failed: Property not found. retry in 115.192024ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.801166182Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\") failed: Property not found. retry in 148.014378ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.815804827Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\") failed: Property not found. retry in 259.002765ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.827494045Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\") failed: Property not found. retry in 67.542249ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.842485075Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\") failed: Property not found. retry in 96.216534ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.855545095Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\") failed: Property not found. retry in 262.87056ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.867393776Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\") failed: Property not found. retry in 100.252139ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.877434107Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\") failed: Property not found. retry in 251.61248ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.888953692Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\") failed: Property not found. retry in 93.265644ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.897828915Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\") failed: Property not found. retry in 64.876282ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.909168997Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\") failed: Property not found. retry in 273.351406ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.92180319Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\") failed: Property not found. retry in 282.336082ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.933231014Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\") failed: Property not found. retry in 171.237972ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.944477994Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\") failed: Property not found. retry in 281.819303ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.956245843Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\") failed: Property not found. retry in 208.669963ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.96660394Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService27\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::c\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte4\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.2.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.2.7\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:55.966843064Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"wait for service Some(\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\") failed: Property not found. retry in 115.864089ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.978032734Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\") failed: Property not found. retry in 130.878342ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:55.988782197Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\") failed: Property not found. retry in 297.84362ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.000685848Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\") failed: Property not found. retry in 203.610641ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.010085379Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\") failed: Property not found. retry in 260.052931ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.020633018Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:19:56.020700359Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\") failed: Property not found. retry in 389.532658ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.099423864Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\") failed: Property not found. retry in 490.648397ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.110240818Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\") failed: Property not found. retry in 497.935824ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.127398084Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\") failed: Property not found. retry in 452.461151ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.140672087Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\") failed: Property not found. retry in 517.418632ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.152880103Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\") failed: Property not found. retry in 216.608549ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.16389231Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\") failed: Property not found. retry in 223.991188ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.187151364Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\") failed: Property not found. retry in 504.430336ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.208706311Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\") failed: Property not found. retry in 219.020344ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.225047913Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\") failed: Property not found. retry in 212.45762ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.235873667Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\") failed: Property not found. retry in 60.590453ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.247182569Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\") failed: Property not found. retry in 308.203253ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.257945592Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\") failed: Property not found. retry in 310.526753ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.292976625Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\") failed: Property not found. retry in 1.102110525s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.326446863Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\") failed: Property not found. retry in 118.740804ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.359626436Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\") failed: Property not found. retry in 1.113216648s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.42212583Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\") failed: Property not found. retry in 593.999058ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.432416355Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\") failed: Property not found. retry in 557.569048ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.454630052Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\") failed: Property not found. retry in 959.814655ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.4650801Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\") failed: Property not found. retry in 185.930396ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.492827476Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\") failed: Property not found. retry in 1.060676585s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.502607454Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\") failed: Property not found. retry in 695.445796ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.58015702Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\") failed: Property not found. retry in 600.138203ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.609801196Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\") failed: Property not found. retry in 1.031405213s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.622972078Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\") failed: Property not found. retry in 408.693674ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.6349377Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\") failed: Property not found. retry in 942.878053ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.646502986Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\") failed: Property not found. retry in 1.099127272s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.682654116Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\") failed: Property not found. retry in 436.53035ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.697166559Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\") failed: Property not found. retry in 859.604295ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:56.725300492Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\") failed: Property not found. retry in 1.201899949s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:57.025030508Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\") failed: Property not found. retry in 780.64482ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:57.039751285Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\") failed: Property not found. retry in 714.547035ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:57.056717017Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\") failed: Property not found. retry in 528.359356ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:57.14778045Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\") failed: Property not found. retry in 965.665696ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:57.222985749Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\") failed: Property not found. retry in 721.658382ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:57.293116296Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\") failed: Property not found. retry in 697.661813ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:57.440008637Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\") failed: Property not found. retry in 1.33372887s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:57.454814684Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\") failed: Property not found. retry in 643.587625ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:57.547133618Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\") failed: Property not found. retry in 741.52368ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:57.573968859Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\") failed: Property not found. retry in 1.145679856s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:57.59701899Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\") failed: Property not found. retry in 505.892819ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:57.608524594Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\") failed: Property not found. retry in 725.413286ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:58.411116582Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\") failed: Property not found. retry in 1.23463611s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:58.47447304Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\") failed: Property not found. retry in 762.821159ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:58.511645787Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\") failed: Property not found. retry in 709.898866ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:58.549193381Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\") failed: Property not found. retry in 1.337485892s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:58.570622885Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\") failed: Property not found. retry in 984.474495ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:58.615561767Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\") failed: Property not found. retry in 1.203815302s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:58.636967791Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\") failed: Property not found. retry in 1.266775972s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:58.648160281Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\") failed: Property not found. retry in 936.366435ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:58.664402952Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\") failed: Property not found. retry in 1.223815746s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:58.679948701Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\") failed: Property not found. retry in 580.60765ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:58.691788571Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\") failed: Property not found. retry in 661.85392ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:58.70350717Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\") failed: Property not found. retry in 1.353562509s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:58.75827522Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\") failed: Property not found. retry in 1.377738452s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:58.803437075Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\") failed: Property not found. retry in 1.342371939s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:59.148767725Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\") failed: Property not found. retry in 844.273583ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:59.271710611Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\") failed: Property not found. retry in 1.263347472s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:59.285664025Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\") failed: Property not found. retry in 1.264148721s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:59.29529497Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\") failed: Property not found. retry in 620.41548ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:59.376342712Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\") failed: Property not found. retry in 1.381985895s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:59.611748525Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\") failed: Property not found. retry in 1.406732958s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:59.620981433Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\") failed: Property not found. retry in 1.430445282s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:59.721222983Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\") failed: Property not found. retry in 593.796501ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:59.854634267Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\") failed: Property not found. retry in 624.019761ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:59.916226307Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\") failed: Property not found. retry in 1.001098958s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:59.927205444Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\") failed: Property not found. retry in 1.095551147s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:19:59.937136663Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\") failed: Property not found. retry in 1.052365581s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:00.011104472Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\") failed: Property not found. retry in 639.354398ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:00.126930573Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\") failed: Property not found. retry in 977.519551ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:00.138175434Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\") failed: Property not found. retry in 1.314217321s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:00.195993683Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\") failed: Property not found. retry in 606.842393ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:00.46091686Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\") failed: Property not found. retry in 914.786461ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:00.502216064Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\") failed: Property not found. retry in 802.368291ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:00.550123804Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\") failed: Property not found. retry in 1.022746427s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:00.559075367Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\") failed: Property not found. retry in 1.49378056s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:00.568868965Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\") failed: Property not found. retry in 751.906412ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:00.776728055Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\") failed: Property not found. retry in 1.135903993s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:00.938593406Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\") failed: Property not found. retry in 759.195913ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:01.003798833Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\") failed: Property not found. retry in 510.796734ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:01.03905369Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\") failed: Property not found. retry in 1.147967483s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:01.048004594Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\") failed: Property not found. retry in 1.180932871s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:01.073071777Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\") failed: Property not found. retry in 1.447903532s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:01.083171019Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\") failed: Property not found. retry in 1.411837939s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:01.123864373Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\") failed: Property not found. retry in 993.607683ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:01.160537612Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\") failed: Property not found. retry in 688.04753ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:01.361946349Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\") failed: Property not found. retry in 1.382583546s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:01.426535147Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\") failed: Property not found. retry in 640.401417ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:01.519574362Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\") failed: Property not found. retry in 910.482324ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:01.539574853Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\") failed: Property not found. retry in 827.799976ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:01.558363925Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\") failed: Property not found. retry in 562.609746ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:01.590489012Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\") failed: Property not found. retry in 1.399756812s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:01.772045978Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\") failed: Property not found. retry in 1.16047482s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:02.057978113Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\") failed: Property not found. retry in 1.499301611s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:02.071737444Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\") failed: Property not found. retry in 863.511721ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:02.084184274Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\") failed: Property not found. retry in 990.781479ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:02.161660039Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\") failed: Property not found. retry in 746.115029ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:02.169091519Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\") failed: Property not found. retry in 648.230505ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:02.176500238Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\") failed: Property not found. retry in 813.310019ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:02.204973955Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\") failed: Property not found. retry in 712.247255ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:02.262356008Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\") failed: Property not found. retry in 648.814383ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:02.394403709Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\") failed: Property not found. retry in 1.137169788s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:02.458857935Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\") failed: Property not found. retry in 1.232302866s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:02.540315094Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\") failed: Property not found. retry in 663.770182ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:02.548708859Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\") failed: Property not found. retry in 908.450858ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:02.818336411Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\") failed: Property not found. retry in 1.354513645s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:02.834632313Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\") failed: Property not found. retry in 547.323469ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:02.92653953Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\") failed: Property not found. retry in 534.731281ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:02.957460187Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\") failed: Property not found. retry in 684.454516ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:02.983536096Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\") failed: Property not found. retry in 1.205588214s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:03.029568276Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\") failed: Property not found. retry in 754.517662ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:03.053810395Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\") failed: Property not found. retry in 1.421851858s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:03.160947227Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\") failed: Property not found. retry in 608.961386ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:03.180042964Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\") failed: Property not found. retry in 692.825656ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:03.222044079Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\") failed: Property not found. retry in 1.014723259s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:03.230338082Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService10\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::12\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_d264dd09-dd1e-4b44-ac20-70d4e8728824/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::12\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"af7b5bbf-b629-4e8b-af6b-d9c193f74f15\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:03.378900639Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:03.379137133Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\") failed: Property not found. retry in 1.454910632s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:03.487691748Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\") failed: Property not found. retry in 42.731332ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:03.503279248Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\") failed: Property not found. retry in 1.403593988s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:03.516550362Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\") failed: Property not found. retry in 109.557121ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:03.557021562Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\") failed: Property not found. retry in 1.206110952s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:03.587540052Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\") failed: Property not found. retry in 29.450315ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:03.609220681Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\") failed: Property not found. retry in 1.037247036s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:03.624049719Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\") failed: Property not found. retry in 95.654849ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:03.648682235Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\") failed: Property not found. retry in 187.192033ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:03.677327614Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\") failed: Property not found. retry in 1.089531974s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:03.690274413Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\") failed: Property not found. retry in 1.390782859s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:03.73927784Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\") failed: Property not found. retry in 281.650225ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:03.75610074Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\") failed: Property not found. retry in 1.361509961s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:03.78531188Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\") failed: Property not found. retry in 997.890904ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:03.802278702Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\") failed: Property not found. retry in 749.473224ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:03.819496339Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\") failed: Property not found. retry in 452.965427ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:03.878706441Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\") failed: Property not found. retry in 917.802317ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:03.926466218Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\") failed: Property not found. retry in 279.728721ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.051009889Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService25\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::a\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte2\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.2.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.2.5\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:04.065686005Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:04.066500133Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService28\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::b\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte1\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.2.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.2.6\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:04.072874251Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:04.075703576Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\") failed: Property not found. retry in 516.207278ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.200095905Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\") failed: Property not found. retry in 54.032887ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.24337275Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\") failed: Property not found. retry in 68.588037ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.250876661Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\") failed: Property not found. retry in 711.349507ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.258829659Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\") failed: Property not found. retry in 1.233087117s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.26639656Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\") failed: Property not found. retry in 114.711408ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.306376263Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\") failed: Property not found. retry in 52.48277ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.328101792Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\") failed: Property not found. retry in 892.767307ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.340529012Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\") failed: Property not found. retry in 971.1752ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.348065183Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\") failed: Property not found. retry in 270.077742ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.388121777Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\") failed: Property not found. retry in 176.902735ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.429131706Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\") failed: Property not found. retry in 1.356166758s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.579673955Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\") failed: Property not found. retry in 700.060981ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.59240689Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\") failed: Property not found. retry in 322.505764ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.618205604Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\") failed: Property not found. retry in 339.159862ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.66898643Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\") failed: Property not found. retry in 1.189202102s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.677275073Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\") failed: Property not found. retry in 1.077270584s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.741882351Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\") failed: Property not found. retry in 1.410109798s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.794792641Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\") failed: Property not found. retry in 1.013121578s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.808872278Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\") failed: Property not found. retry in 714.643855ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.824620031Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\") failed: Property not found. retry in 1.28657675s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.855602048Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\") failed: Property not found. retry in 1.062546128s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.930938039Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\") failed: Property not found. retry in 1.146305878s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.952034548Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\") failed: Property not found. retry in 940.610313ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.961905017Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\") failed: Property not found. retry in 1.395623307s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:04.98015607Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\") failed: Property not found. retry in 669.080284ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:05.019689265Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\") failed: Property not found. retry in 583.153472ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:05.141705046Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\") failed: Property not found. retry in 560.697184ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:05.156715207Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\") failed: Property not found. retry in 1.046895329s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:05.242910012Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\") failed: Property not found. retry in 991.577441ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:05.302015682Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\") failed: Property not found. retry in 551.005745ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:05.329733948Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\") failed: Property not found. retry in 1.258291525s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:05.508844146Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\") failed: Property not found. retry in 799.709458ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:05.548816108Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\") failed: Property not found. retry in 629.605434ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:05.700508415Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\") failed: Property not found. retry in 543.566535ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:05.727104863Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\") failed: Property not found. retry in 539.839705ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:05.735533438Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\") failed: Property not found. retry in 603.293975ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:05.829078481Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\") failed: Property not found. retry in 990.783073ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:05.83711176Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\") failed: Property not found. retry in 1.255002645s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:05.87813183Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\") failed: Property not found. retry in 863.951874ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:05.891732978Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\") failed: Property not found. retry in 635.957037ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:05.91615233Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\") failed: Property not found. retry in 1.268419077s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:05.946134022Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\") failed: Property not found. retry in 1.255917397s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:06.00569656Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\") failed: Property not found. retry in 958.389502ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:06.111885946Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\") failed: Property not found. retry in 811.855961ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:06.153053197Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\") failed: Property not found. retry in 1.012667115s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:06.215322768Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\") failed: Property not found. retry in 894.172002ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:06.281788906Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\") failed: Property not found. retry in 1.031493845s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:06.292075447Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\") failed: Property not found. retry in 972.446368ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:06.301555544Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\") failed: Property not found. retry in 1.200988053s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:06.309286038Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\") failed: Property not found. retry in 1.193502632s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:06.339123478Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\") failed: Property not found. retry in 1.490017162s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:06.358867115Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\") failed: Property not found. retry in 674.965668ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:06.386510979Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\") failed: Property not found. retry in 799.64218ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:06.442762353Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\") failed: Property not found. retry in 768.891522ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:06.564132953Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\") failed: Property not found. retry in 761.559555ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:06.770730833Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\") failed: Property not found. retry in 699.206363ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:06.784146588Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\") failed: Property not found. retry in 875.284654ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:06.83774811Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\") failed: Property not found. retry in 928.266333ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:06.976374187Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\") failed: Property not found. retry in 1.415822022s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:07.073689411Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\") failed: Property not found. retry in 808.544763ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:07.084284472Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\") failed: Property not found. retry in 1.03160745s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:07.143744547Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\") failed: Property not found. retry in 827.339561ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:07.189607514Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\") failed: Property not found. retry in 599.510153ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:07.227426382Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\") failed: Property not found. retry in 970.623174ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:07.239067759Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\") failed: Property not found. retry in 1.128156402s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:07.254367035Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\") failed: Property not found. retry in 991.50096ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:07.274864684Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\") failed: Property not found. retry in 789.515478ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:07.286845156Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\") failed: Property not found. retry in 556.99196ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:07.337786275Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\") failed: Property not found. retry in 1.134159367s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:07.352869297Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\") failed: Property not found. retry in 848.139753ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:07.49805141Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\") failed: Property not found. retry in 803.085794ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:07.523676332Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\") failed: Property not found. retry in 1.439995287s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:07.547766189Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\") failed: Property not found. retry in 875.149099ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:07.561199195Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\") failed: Property not found. retry in 1.485422123s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:07.724502019Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\") failed: Property not found. retry in 1.136178962s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:07.840418882Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\") failed: Property not found. retry in 670.3088ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:07.862377165Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\") failed: Property not found. retry in 560.522461ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:07.907852246Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\") failed: Property not found. retry in 1.467396801s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:07.918179781Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\") failed: Property not found. retry in 1.443290494s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:07.928590129Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\") failed: Property not found. retry in 1.010073361s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:08.042656542Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService19\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::e\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/clickhouse\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::e\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"8123\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:08.052386458Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:08.052600452Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\") failed: Property not found. retry in 844.513459ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:08.092180148Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\") failed: Property not found. retry in 1.18959512s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:08.189976669Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\") failed: Property not found. retry in 925.35021ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:08.224506394Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\") failed: Property not found. retry in 918.570926ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:08.282180311Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\") failed: Property not found. retry in 1.29964706s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:08.341069377Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\") failed: Property not found. retry in 62.050773ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:08.351752089Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\") failed: Property not found. retry in 644.788204ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:08.361197921Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\") failed: Property not found. retry in 843.91479ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:08.392122018Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\") failed: Property not found. retry in 98.620448ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:08.424456117Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\") failed: Property not found. retry in 537.756676ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:08.452058581Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\") failed: Property not found. retry in 838.768647ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:08.691704801Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\") failed: Property not found. retry in 599.712852ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:08.703673934Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\") failed: Property not found. retry in 1.027167239s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:08.713723855Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\") failed: Property not found. retry in 771.482452ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:08.72398864Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\") failed: Property not found. retry in 278.899677ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:08.735153599Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\") failed: Property not found. retry in 1.395544869s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:08.946636668Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\") failed: Property not found. retry in 518.844644ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:08.988380338Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\") failed: Property not found. retry in 1.102018164s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:09.000797628Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\") failed: Property not found. retry in 1.23148164s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:09.013194497Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\") failed: Property not found. retry in 1.097941129s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:09.024985067Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\") failed: Property not found. retry in 529.224811ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:09.036437691Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\") failed: Property not found. retry in 956.613009ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:09.062149774Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\") failed: Property not found. retry in 1.076641175s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:09.164540099Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\") failed: Property not found. retry in 1.052547881s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:09.213970304Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\") failed: Property not found. retry in 1.192388664s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:09.231112429Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\") failed: Property not found. retry in 993.60104ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:09.256280674Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\") failed: Property not found. retry in 1.228697809s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:09.326142706Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\") failed: Property not found. retry in 572.38608ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:09.404519606Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\") failed: Property not found. retry in 1.394733861s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:09.427359793Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\") failed: Property not found. retry in 1.027678281s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:09.439899764Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\") failed: Property not found. retry in 1.25576303s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:09.519575955Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\") failed: Property not found. retry in 572.931468ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:09.541237713Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\") failed: Property not found. retry in 1.18488331s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:09.558693223Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\") failed: Property not found. retry in 909.259496ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:09.579822433Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\") failed: Property not found. retry in 693.103988ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:09.657879887Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\") failed: Property not found. retry in 974.26532ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:09.756106705Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\") failed: Property not found. retry in 1.247339613s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:09.988553721Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\") failed: Property not found. retry in 654.022693ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:10.029305196Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\") failed: Property not found. retry in 1.423563257s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:10.11671269Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\") failed: Property not found. retry in 580.745437ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:10.127017926Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\") failed: Property not found. retry in 970.500331ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:10.135809397Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\") failed: Property not found. retry in 1.10093237s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:10.254003356Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\") failed: Property not found. retry in 986.854783ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:10.263922196Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\") failed: Property not found. retry in 984.305912ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:10.274527346Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\") failed: Property not found. retry in 1.383251378s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:10.283072974Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\") failed: Property not found. retry in 1.398758886s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:10.354526152Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\") failed: Property not found. retry in 1.127887242s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:10.366284231Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\") failed: Property not found. retry in 1.29479035s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:10.439916144Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\") failed: Property not found. retry in 813.564564ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:10.48135927Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\") failed: Property not found. retry in 1.058981235s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:10.508190111Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\") failed: Property not found. retry in 1.438880027s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:10.570737216Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\") failed: Property not found. retry in 629.156986ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:10.704753679Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\") failed: Property not found. retry in 763.141282ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:10.723397409Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\") failed: Property not found. retry in 1.058689734s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:10.743743206Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\") failed: Property not found. retry in 1.354198615s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:10.754410377Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\") failed: Property not found. retry in 558.138538ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:10.793915952Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\") failed: Property not found. retry in 1.347666136s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:10.834744028Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\") failed: Property not found. retry in 1.468862078s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:11.121818641Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\") failed: Property not found. retry in 547.158113ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:11.252248537Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\") failed: Property not found. retry in 1.262001325s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:11.268939265Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\") failed: Property not found. retry in 1.081928557s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:11.282416642Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\") failed: Property not found. retry in 951.807079ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:11.309787372Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\") failed: Property not found. retry in 618.030272ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:11.348534675Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\") failed: Property not found. retry in 1.058397349s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:11.363305452Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\") failed: Property not found. retry in 1.117014528s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:11.373986113Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\") failed: Property not found. retry in 1.272026363s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:11.501460242Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\") failed: Property not found. retry in 794.93106ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:11.513926052Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\") failed: Property not found. retry in 1.249886274s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:11.551438185Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\") failed: Property not found. retry in 1.144832589s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:11.578523951Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\") failed: Property not found. retry in 1.192290344s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:11.67872868Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\") failed: Property not found. retry in 948.013109ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:11.74840429Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\") failed: Property not found. retry in 779.106919ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:11.765287941Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\") failed: Property not found. retry in 1.247159624s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:11.812405698Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\") failed: Property not found. retry in 960.337049ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:11.823843952Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a\") failed: Property not found. retry in 1.20625256s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:11.981242811Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7\") failed: Property not found. retry in 1.492598593s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:12.025227278Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\") failed: Property not found. retry in 565.67716ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:12.122029014Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a\") failed: Property not found. retry in 1.40132011s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:12.196357598Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\") failed: Property not found. retry in 706.397209ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:12.274088607Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\") failed: Property not found. retry in 776.108242ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:12.323700215Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\") failed: Property not found. retry in 1.179198909s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:12.379251697Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff\") failed: Property not found. retry in 1.360456005s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:12.436438406Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5\") failed: Property not found. retry in 979.936398ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:12.507494018Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9\") failed: Property not found. retry in 890.877164ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:12.543810182Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\") failed: Property not found. retry in 629.671475ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:12.559304621Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813\") failed: Property not found. retry in 1.267106879s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:12.603147955Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283\") failed: Property not found. retry in 1.405127057s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:12.705901416Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f\") failed: Property not found. retry in 658.930545ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:12.720084534Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\") failed: Property not found. retry in 965.424094ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:12.736570309Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d\") failed: Property not found. retry in 854.206444ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:12.790516246Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\") failed: Property not found. retry in 1.176968816s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:12.805359584Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\") failed: Property not found. retry in 537.002781ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:12.818436434Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197\") failed: Property not found. retry in 503.961848ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:12.889852492Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc\") failed: Property not found. retry in 1.394981359s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:13.003775353Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019\") failed: Property not found. retry in 925.601798ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:13.089922417Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3\") failed: Property not found. retry in 1.195189003s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:13.126428034Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8\") failed: Property not found. retry in 1.246550187s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:13.247854875Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887\") failed: Property not found. retry in 1.003635954s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:13.859967911Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\") failed: Property not found. retry in 583.151248ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:14.018759712Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96\") failed: Property not found. retry in 1.316960888s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:14.183332137Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\") failed: Property not found. retry in 1.36465581s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:14.685709469Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\") failed: Property not found. retry in 1.087327878s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:16.070183837Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5\") failed: Property not found. retry in 1.297410541s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:20:17.169948989Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.495145755Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_4bd02937-0d24-4783-b773-973d42c4eef8"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.49547988Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_14af2360-4bb4-4f0c-88c4-c36ebd6f02c7"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.49549073Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_c8b70fc0-b959-4973-889c-bcdff32c973f"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.49549649Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_oximeter_ea63e896-0dc6-4da7-b3d5-27de7463f89a"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.49550228Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_bb2cdb6e-aac0-4124-a13e-9c3e19120ba0"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.49550736Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_652e3013-976a-43ba-bb68-b6f497ea4197"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.49551264Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_a64df5e3-8d68-4581-8828-def85de45a96"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.495517901Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_4dfbb131-ca76-448d-8328-3ae1e5fa72b9"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.495523261Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_82e8a1b2-fbe0-4447-944a-27decf04f25a"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.495528591Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_a1c22271-87f5-44b0-b506-06dff6539283"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.495538111Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_0ff6eb74-346e-45ea-890b-3a8a4237dcff"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.495546781Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_clickhouse_ff8af3cb-0e96-47e9-b05c-afa245f1dff5"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.495552601Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_external_dns_350afff8-3a4b-48f9-b954-9cd28e0a2019"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.495568151Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_2d696101-87c5-4c96-b623-2e415a336b1d"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.495573781Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_34579d2e-b8f0-4cf3-bcf0-ba9d8c19e7fc"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.495578981Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_d78f8b7a-b7a9-41c5-83d8-2b2ed0991813"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.495584362Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_external_dns_41ef6d1d-d04a-4944-8844-640cc5b2f887"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.495593092Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.495598642Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_43c9638c-6930-4233-a47d-c58f94e587c5"}
{"msg":"Writing ledger to /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.496666289Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.498275465Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.5023608Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"4f5fea2a-a83f-4174-a588-d45e027c8dba","remote_addr":"[fd00:1122:3344:101::1]:34328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":41488745,"response_code":"204"}
{"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.512688346Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1212"}
{"msg":"Writing ledger to /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.51289604Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.513049052Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.515255758Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:679"}
{"msg":"Nexus address: [fd00:1122:3344:101::c]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.516076091Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:692"}
{"msg":"rack_network_config: RackNetworkConfigV1 {\n    bfd: [],\n    bgp: [],\n    infra_ip_first: 10.151.2.174,\n    infra_ip_last: 10.151.2.174,\n    ports: [\n        PortConfigV1 {\n            addresses: [\n                V4(\n                    Ipv4Network {\n                        addr: 10.151.2.174,\n                        prefix: 24,\n                    },\n                ),\n            ],\n            autoneg: false,\n            bgp_peers: [],\n            port: \"qsfp0\",\n            routes: [\n                RouteConfig {\n                    destination: V4(\n                        Ipv4Network {\n                            addr: 0.0.0.0,\n                            prefix: 0,\n                        },\n                    ),\n                    nexthop: 10.151.2.100,\n                },\n            ],\n            switch: Switch0,\n            uplink_port_fec: None,\n            uplink_port_speed: Speed40G,\n        },\n    ],\n    rack_subnet: Ipv6Network {\n        addr: fd00:1122:3344:100::,\n        prefix: 56,\n    },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.54836168Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:803"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/620b992f-d3de-45ff-bbb4-3439178676d0/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.552171631Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/620b992f-d3de-45ff-bbb4-3439178676d0/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.662997051Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/620b992f-d3de-45ff-bbb4-3439178676d0/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.842578907Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/620b992f-d3de-45ff-bbb4-3439178676d0/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:18.952144638Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/620b992f-d3de-45ff-bbb4-3439178676d0/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:19.084493554Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/620b992f-d3de-45ff-bbb4-3439178676d0/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:19.246083391Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/620b992f-d3de-45ff-bbb4-3439178676d0/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:19.549472276Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/620b992f-d3de-45ff-bbb4-3439178676d0/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:19.799801339Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/620b992f-d3de-45ff-bbb4-3439178676d0/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:20.152185702Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/620b992f-d3de-45ff-bbb4-3439178676d0/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:20.568563702Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/620b992f-d3de-45ff-bbb4-3439178676d0/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:21.023910749Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/620b992f-d3de-45ff-bbb4-3439178676d0/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:21.508102465Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:21.962836527Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35294"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:21.962957839Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"377eeea6-8848-4211-aa4e-0f6eb629b4d9","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35294","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":54,"response_code":"200"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/620b992f-d3de-45ff-bbb4-3439178676d0/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:22.196171267Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/620b992f-d3de-45ff-bbb4-3439178676d0/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:22.842145636Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/620b992f-d3de-45ff-bbb4-3439178676d0/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:24.381617565Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"}
{"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fa9f8d8b-ab21-41f6-b059-f116a7fe0ca4\", \"content-length\": \"123\", \"date\": \"Fri, 26 Apr 2024 00:20:25 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"rack not found\", request_id: \"fa9f8d8b-ab21-41f6-b059-f116a7fe0ca4\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:25.29923325Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"}
{"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a881b869-8d9b-47d0-829f-7fa0c8ca68a4\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:20:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a881b869-8d9b-47d0-829f-7fa0c8ca68a4\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:26.317494082Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"}
{"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5178b094-b695-43e5-9d97-ad2722cb51d3\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:20:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5178b094-b695-43e5-9d97-ad2722cb51d3\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:27.703642796Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"}
{"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"51f6297a-0ec1-40cb-87a9-0c25b3e48731\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:20:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"51f6297a-0ec1-40cb-87a9-0c25b3e48731\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:29.174409041Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"}
{"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ec73982b-bec8-4ce2-85ae-f8157b09485a\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:20:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ec73982b-bec8-4ce2-85ae-f8157b09485a\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:30.818703143Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"}
{"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"41c2e792-4fa4-499b-b542-166f681c83b8\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:20:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"41c2e792-4fa4-499b-b542-166f681c83b8\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:32.9213112Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"}
{"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1ed79cc0-4bcd-4f69-99c7-5b52219b9530\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:20:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1ed79cc0-4bcd-4f69-99c7-5b52219b9530\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:37.394164679Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"}
{"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9ae77cc9-311b-44d9-9647-aacbc8a133b2\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:20:40 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9ae77cc9-311b-44d9-9647-aacbc8a133b2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:40.146388766Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"}
{"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5e793803-296b-47f8-8970-f3d948296308\", \"content-length\": \"124\", \"date\": \"Fri, 26 Apr 2024 00:20:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5e793803-296b-47f8-8970-f3d948296308\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:47.325778714Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:51.992631693Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45607"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:51.992735825Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"2e6c7454-54c3-49df-b005-3dcf4ce8d386","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45607","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":62,"response_code":"200"}
{"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:54.513086498Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"779d9354-af5f-4248-aa8f-b97427c8939e","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:54.657581269Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:41411"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:54.660285532Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:54.749586007Z","hostname":"sock","pid":1590,"uri":"/inventory","method":"GET","req_id":"d8bce79f-af7c-4cc6-9c26-650d0a523ba0","remote_addr":"[fd00:1122:3344:101::a]:41411","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":91927,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:54.750332879Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:54.75041367Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:54.753911816Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:54.755155247Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"GET","req_id":"3d4ee534-500d-4c21-8ee8-920d3033be29","remote_addr":"[fd00:1122:3344:101::a]:41411","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":4953,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:55.924729138Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:36730"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:55.926314153Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"ec74a417-e0fb-44b1-8e17-09ba7f5c3667","remote_addr":"[fd00:1122:3344:101::c]:36730","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1561,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:56.853755773Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:48031"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:56.855612903Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 34398e66-3ca2-4650-99bf-308c5bfd1e01, kind: Service { id: 350afff8-3a4b-48f9-b954-9cd28e0a2019 }, name: Name(\"external-dns-350afff8-3a4b-48f9-b954-9cd28e0a2019\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 70c6ff6d-2499-4301-899d-eb9940b9509f, kind: Service { id: 41ef6d1d-d04a-4944-8844-640cc5b2f887 }, name: Name(\"external-dns-41ef6d1d-d04a-4944-8844-640cc5b2f887\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 22522ec7-9548-4589-9ee2-1008d71e9856, kind: Service { id: bb2cdb6e-aac0-4124-a13e-9c3e19120ba0 }, name: Name(\"nexus-bb2cdb6e-aac0-4124-a13e-9c3e19120ba0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 67])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 66fe50d8-5616-40d8-8088-4a08f64aa8a3, kind: Service { id: a1c22271-87f5-44b0-b506-06dff6539283 }, name: Name(\"nexus-a1c22271-87f5-44b0-b506-06dff6539283\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 0])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c3873700-f1d1-4d2b-8984-a596fed1d2db, kind: Service { id: 2d696101-87c5-4c96-b623-2e415a336b1d }, name: Name(\"nexus-2d696101-87c5-4c96-b623-2e415a336b1d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:56.855666424Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:56.855706965Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:56.855714735Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:56.855723185Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:56.855732555Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:56.855756936Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:56.855776566Z","hostname":"sock","pid":1590,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"02fad422-e156-4503-9274-11adad861b1b","remote_addr":"[fd00:1122:3344:101::c]:48031","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1514,"response_code":"204"}
{"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:56.858047413Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:865"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:57.082173474Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:60117"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:57.083299492Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:57.171947716Z","hostname":"sock","pid":1590,"uri":"/inventory","method":"GET","req_id":"a7ae42a5-2cac-48fe-83cb-2ab132fd1202","remote_addr":"[fd00:1122:3344:101::c]:60117","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":89665,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:57.172903601Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:57.172967442Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:57.17341212Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:20:57.173687674Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"GET","req_id":"94698cc5-11cf-44ed-a2cd-e2315bc8fca7","remote_addr":"[fd00:1122:3344:101::c]:60117","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1473,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:22.025141297Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48337"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:22.02531576Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"7b0f1fff-6b2a-4b07-a8b8-7b1cd829c7bf","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48337","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":148,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:24.062093413Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::d]:65007"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:24.070776183Z","hostname":"sock","pid":1590,"uri":"/metrics/collect/779d9354-af5f-4248-aa8f-b97427c8939e","method":"GET","req_id":"0d81fe22-90f8-454f-b59d-0c2546120761","remote_addr":"[fd00:1122:3344:101::d]:65007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8651,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:25.044237563Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:39234"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:25.130682632Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"4927656b-7ec9-4f6d-abe5-ae0f6ffec0cb","remote_addr":"[fd00:1122:3344:101::c]:39234","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":86399,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:25.130800014Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:37047"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:25.183938878Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"dc630a9f-dab7-49cc-873a-d90c209b8419","remote_addr":"[fd00:1122:3344:101::b]:37047","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":52916,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:25.184235562Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"178fcbbb-fe31-4b19-a033-9fbf15d66bd7","remote_addr":"[fd00:1122:3344:101::b]:37047","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":117,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:25.184323454Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"4589fb1d-cd9f-4903-abd9-833a58fe48d5","remote_addr":"[fd00:1122:3344:101::c]:39234","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":173,"response_code":"200"}
{"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:25.209824914Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"}
{"msg":"Writing ledger to /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:25.211120314Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:25.212552877Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:25.213745347Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:25.213772197Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"PUT","req_id":"8f22818a-6d28-4dc2-b647-5bba0bdc139b","remote_addr":"[fd00:1122:3344:101::c]:39234","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":4518,"response_code":"204"}
{"msg":"Attempting to update network config with generation: 3, current_generation: 2","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:25.216386439Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"}
{"msg":"Writing ledger to /pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:25.21642842Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:25.216584192Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Broadcasting network config with generation 3","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:25.216688824Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:25.216698264Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"PUT","req_id":"5745cb04-99ca-4906-9313-c9f5e0e11d17","remote_addr":"[fd00:1122:3344:101::b]:37047","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":414,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:25.379827525Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:52253"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:25.42247435Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"b6fe442f-a448-426b-aa03-68f433c06efc","remote_addr":"[fd00:1122:3344:101::a]:52253","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":42610,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:25.422766415Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"85eba420-14bc-4d13-bb55-68dadd99bd22","remote_addr":"[fd00:1122:3344:101::a]:52253","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":94,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:52.055814539Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37584"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:52.055974662Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"ecc18612-b140-4ebf-ac8b-92463333125d","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37584","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":117,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:54.062005141Z","hostname":"sock","pid":1590,"uri":"/metrics/collect/779d9354-af5f-4248-aa8f-b97427c8939e","method":"GET","req_id":"a16c0012-bbc5-4176-b82a-d5ab9e06f852","remote_addr":"[fd00:1122:3344:101::d]:65007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":747,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:54.819277517Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:46544"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:54.866678519Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"c8dbd0db-8625-47c3-91a2-6e188de60df6","remote_addr":"[fd00:1122:3344:101::b]:46544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47330,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:54.866979653Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"0c1e4776-1788-4abd-8b16-124754fbcfa3","remote_addr":"[fd00:1122:3344:101::b]:46544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":98,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:54.889294732Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:48451"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:54.935658027Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"5a1b5b09-9df9-401f-816d-917683086e69","remote_addr":"[fd00:1122:3344:101::c]:48451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46262,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:54.935934592Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"3aadf76f-e403-4970-88bb-f99590c5fe6b","remote_addr":"[fd00:1122:3344:101::c]:48451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":94,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:55.192854549Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:65522"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:55.240674057Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"615673c7-43a5-4115-843f-b848ebeb273f","remote_addr":"[fd00:1122:3344:101::a]:65522","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47733,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:21:55.240958742Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"29d40ccc-fdc7-4c10-9f65-1c3274109134","remote_addr":"[fd00:1122:3344:101::a]:65522","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":97,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:22.090388259Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59046"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:22.091168822Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"9ed015fd-7856-4879-86bb-74482e47e263","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59046","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":56,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:24.062644726Z","hostname":"sock","pid":1590,"uri":"/metrics/collect/779d9354-af5f-4248-aa8f-b97427c8939e","method":"GET","req_id":"f17ed6ca-316a-4324-94b9-64d5914ec71f","remote_addr":"[fd00:1122:3344:101::d]:65007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1719,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:25.058034958Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:63898"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:25.109425533Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"206bdb72-b7fb-4d5e-9f02-b73ce020deb0","remote_addr":"[fd00:1122:3344:101::b]:63898","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":51343,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:25.109559766Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:37355"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:25.113302816Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"ad29be26-85d3-4c7c-8c82-f2fca0febfcd","remote_addr":"[fd00:1122:3344:101::b]:63898","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3543,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:25.153692375Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"383a8c66-2125-405d-b4c8-64641dbcbef8","remote_addr":"[fd00:1122:3344:101::c]:37355","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44033,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:25.1540453Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"9bd1b406-c126-4bbc-a61e-d61254940f77","remote_addr":"[fd00:1122:3344:101::c]:37355","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":155,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:25.507085133Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:34240"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:25.549113148Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"187b1040-a5b1-4bc1-9a84-f5f5d5d18828","remote_addr":"[fd00:1122:3344:101::a]:34240","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":41960,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:25.549940311Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"26f46f1e-7858-4e21-bef4-f174146ae128","remote_addr":"[fd00:1122:3344:101::a]:34240","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":665,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:52.124355955Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39915"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:52.124478847Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"e661b850-4cba-4423-b712-77f6d086f0ea","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39915","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":107,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:54.061473446Z","hostname":"sock","pid":1590,"uri":"/metrics/collect/779d9354-af5f-4248-aa8f-b97427c8939e","method":"GET","req_id":"a0180860-43f2-4588-861b-d8155ea351d0","remote_addr":"[fd00:1122:3344:101::d]:65007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":721,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:54.840526172Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:60461"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:54.8889772Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"36e6c6b6-0533-4c03-ae86-4036c1957a5f","remote_addr":"[fd00:1122:3344:101::b]:60461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48386,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:54.889114272Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:51831"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:54.892493586Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"a1396fc9-6c8a-4af0-835b-ae957700b51c","remote_addr":"[fd00:1122:3344:101::b]:60461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3164,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:54.935897714Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"70998584-bcdc-4a4f-bce0-9c4e25c158db","remote_addr":"[fd00:1122:3344:101::c]:51831","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46666,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:54.936245709Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"65da5576-6cc7-4277-b247-0fa2720f9b61","remote_addr":"[fd00:1122:3344:101::c]:51831","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":144,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:55.373793269Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:39967"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:55.421828601Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"0c2f54f4-efda-46b8-b63d-e5aed597e694","remote_addr":"[fd00:1122:3344:101::a]:39967","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47966,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:55.422091315Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"da580a47-d152-41d3-b0c0-b8461bbf062d","remote_addr":"[fd00:1122:3344:101::a]:39967","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":102,"response_code":"200"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:22:57.36818087Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:23:22.15729697Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61775"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:23:22.157402982Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"656f483c-5f5f-456e-8505-fd7cf3a385ff","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61775","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":78,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:23:24.062735192Z","hostname":"sock","pid":1590,"uri":"/metrics/collect/779d9354-af5f-4248-aa8f-b97427c8939e","method":"GET","req_id":"5680658c-d7c3-44ae-a19b-27c68bdcb8ea","remote_addr":"[fd00:1122:3344:101::d]:65007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":576,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:23:25.031261741Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:33873"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:23:25.081688051Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"ad40f15d-ac76-439f-8767-f42f5e82c6a0","remote_addr":"[fd00:1122:3344:101::c]:33873","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50340,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:23:25.081805764Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:59284"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:23:25.125994414Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"726288b7-e334-4a3c-814f-3b002951d3ad","remote_addr":"[fd00:1122:3344:101::b]:59284","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44114,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:23:25.126838856Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"94e7c5dd-1285-4c68-903d-775bfd19f86c","remote_addr":"[fd00:1122:3344:101::c]:33873","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":760,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:23:25.127113904Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"1bfeab55-ed9e-40c3-8e92-4a5e7de7624c","remote_addr":"[fd00:1122:3344:101::b]:59284","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":184,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:23:25.759769055Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:60878"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:23:25.806239462Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"7e9da928-86a9-447c-a260-f72cd54fed0a","remote_addr":"[fd00:1122:3344:101::a]:60878","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46430,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:23:25.806470705Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"c2484a42-d610-4126-8d95-272ce87e1d4e","remote_addr":"[fd00:1122:3344:101::a]:60878","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":93,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:23:52.200166344Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39257"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:23:52.200364297Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"c9c6d174-7fa0-406a-87ce-cb8fc7f1c423","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39257","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":160,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:23:54.063604481Z","hostname":"sock","pid":1590,"uri":"/metrics/collect/779d9354-af5f-4248-aa8f-b97427c8939e","method":"GET","req_id":"966d421c-0095-4ce8-b60f-30db7a6d6ef1","remote_addr":"[fd00:1122:3344:101::d]:65007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":600,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:23:54.907970786Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:41058"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:23:54.958060771Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"4b064541-2442-462f-b12e-2366bd54e0bc","remote_addr":"[fd00:1122:3344:101::c]:41058","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50038,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:23:54.958306995Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:55219"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:23:54.958400464Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"45065391-956a-472f-a5c5-730e77ac9be0","remote_addr":"[fd00:1122:3344:101::c]:41058","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":142,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:23:55.006373357Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"635988e6-bf3d-4825-a972-4855f0480cb4","remote_addr":"[fd00:1122:3344:101::b]:55219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47914,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:23:55.006781304Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"21a535c0-c3d1-4f1e-ab59-582f998b3c90","remote_addr":"[fd00:1122:3344:101::b]:55219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":132,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:23:55.168336329Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:60216"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:23:55.214402999Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"8d4792c6-043c-4d63-a8a0-eaf282166fc9","remote_addr":"[fd00:1122:3344:101::a]:60216","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46004,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:23:55.215355044Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"3cb7a901-e27a-44da-9ff1-36e10b9ff088","remote_addr":"[fd00:1122:3344:101::a]:60216","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":782,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:24:22.231878179Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40961"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:24:22.2319974Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"57c614ac-0dff-4c76-a229-e639d89bab19","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40961","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":97,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:24:24.064963728Z","hostname":"sock","pid":1590,"uri":"/metrics/collect/779d9354-af5f-4248-aa8f-b97427c8939e","method":"GET","req_id":"376dae22-6c2c-416f-a39a-65b5985811a0","remote_addr":"[fd00:1122:3344:101::d]:65007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1543,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:24:25.061856134Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:57598"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:24:25.109528349Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"ae1c6292-0217-4ebb-81fb-fef977883b68","remote_addr":"[fd00:1122:3344:101::c]:57598","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47621,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:24:25.110483895Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"1412f23e-0ee0-44a7-8cb5-d58c5ffaa019","remote_addr":"[fd00:1122:3344:101::c]:57598","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":809,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:24:25.12947418Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:45760"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:24:25.176907232Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"ace588f6-a038-4f9f-b7c6-426111c01674","remote_addr":"[fd00:1122:3344:101::b]:45760","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47353,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:24:25.177193707Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"a7cd69c9-43cd-49e4-82bd-4a813d93d7ef","remote_addr":"[fd00:1122:3344:101::b]:45760","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":110,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:24:25.533400619Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:40604"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:24:25.580410235Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"64aec66c-7044-4360-9cc0-76322acda080","remote_addr":"[fd00:1122:3344:101::a]:40604","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46965,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:24:25.580644978Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"3ad4db42-dc55-4535-9564-bee9ed3d832d","remote_addr":"[fd00:1122:3344:101::a]:40604","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":89,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:24:52.26584171Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53743"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:24:52.265980362Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"b3b46e83-4af3-4223-80c2-5d102ff66bac","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53743","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":106,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:24:54.064911313Z","hostname":"sock","pid":1590,"uri":"/metrics/collect/779d9354-af5f-4248-aa8f-b97427c8939e","method":"GET","req_id":"bb72e302-f46a-46c4-bedc-78d87a2d4684","remote_addr":"[fd00:1122:3344:101::d]:65007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":530,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:24:54.849347975Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:56338"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:24:54.900003539Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"c7e97bfd-662e-4476-8cd9-8866bf4dbe7b","remote_addr":"[fd00:1122:3344:101::b]:56338","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50591,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:24:54.900298394Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"a5792791-7d15-48ce-a3b0-6351911afbf5","remote_addr":"[fd00:1122:3344:101::b]:56338","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":100,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:24:54.977727218Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:37375"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:24:55.026894427Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"58f5378d-6fe6-49b7-9369-5ff20130f356","remote_addr":"[fd00:1122:3344:101::c]:37375","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49105,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:24:55.027151792Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"5a528163-5f99-414e-a5b1-6ced96a379d9","remote_addr":"[fd00:1122:3344:101::c]:37375","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":90,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:24:55.22567969Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:36858"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:24:55.274028128Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"c3168054-17b5-464a-9dd8-563950efba50","remote_addr":"[fd00:1122:3344:101::a]:36858","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48229,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:24:55.274278152Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"a561979a-c0b8-44b9-a460-f3de178e027f","remote_addr":"[fd00:1122:3344:101::a]:36858","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":94,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:25:22.299265195Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64944"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:25:22.299910786Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"b16ecad4-18da-47a6-9fb9-3428878930bc","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64944","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":616,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:25:24.066503365Z","hostname":"sock","pid":1590,"uri":"/metrics/collect/779d9354-af5f-4248-aa8f-b97427c8939e","method":"GET","req_id":"77e78d7e-dd55-422c-98a7-4f15a0692dda","remote_addr":"[fd00:1122:3344:101::d]:65007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":599,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:25:25.050123607Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:47409"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:25:25.099559411Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"f6633d7c-2fcf-48a8-b57c-396b92262761","remote_addr":"[fd00:1122:3344:101::b]:47409","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49338,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:25:25.099657983Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:55172"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:25:25.14305049Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"4723e33c-8582-453f-a88b-6712f10b5e20","remote_addr":"[fd00:1122:3344:101::c]:55172","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43331,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:25:25.143269134Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"4f47d143-5e95-4866-92d3-5a117961fc2c","remote_addr":"[fd00:1122:3344:101::b]:47409","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":108,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:25:25.143492847Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"7948cdfb-e97c-4e16-9219-ea3463db0b7c","remote_addr":"[fd00:1122:3344:101::c]:55172","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":147,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:25:25.667557056Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:59854"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:25:25.709991238Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"0a2f0ade-2cad-4a6a-9f71-028f79b7de6a","remote_addr":"[fd00:1122:3344:101::a]:59854","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":42373,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:25:25.710759Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"4bbe13bd-f2d9-4315-996c-dcc6ee1cafe6","remote_addr":"[fd00:1122:3344:101::a]:59854","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":626,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:25:52.337701422Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57719"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:25:52.337787373Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"bc7da71d-3da1-4115-b3df-5278aac88f7f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57719","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":86,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:25:54.073708691Z","hostname":"sock","pid":1590,"uri":"/metrics/collect/779d9354-af5f-4248-aa8f-b97427c8939e","method":"GET","req_id":"a22c946d-f281-4d2f-9984-4f88fa17e8f0","remote_addr":"[fd00:1122:3344:101::d]:65007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":596,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:25:54.820774531Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:50745"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:25:54.8693053Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"c11be643-1e8f-4256-99e7-f81cf714a443","remote_addr":"[fd00:1122:3344:101::b]:50745","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48466,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:25:54.869576384Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"be7e1914-784a-4011-bbd0-4430582e22e8","remote_addr":"[fd00:1122:3344:101::b]:50745","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":110,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:25:54.956973598Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:40152"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:25:55.003953523Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"d85ce264-c493-4128-aa0a-b84d5788d21b","remote_addr":"[fd00:1122:3344:101::c]:40152","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46892,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:25:55.004188097Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"de3b1cd5-7e24-4dc5-9eb6-fb2c885172c4","remote_addr":"[fd00:1122:3344:101::c]:40152","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":84,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:25:55.411169174Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:61763"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:25:55.458273081Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"3c906818-9de7-4f6b-b118-451f1ca6d770","remote_addr":"[fd00:1122:3344:101::a]:61763","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47035,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:25:55.459004829Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"7af5c643-a5d3-432a-8b52-7c61c8952ba9","remote_addr":"[fd00:1122:3344:101::a]:61763","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":605,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:26:22.374730882Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47176"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:26:22.374829784Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"f7332081-df80-49be-9d9d-cb738cc448fa","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47176","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":78,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:26:24.076171027Z","hostname":"sock","pid":1590,"uri":"/metrics/collect/779d9354-af5f-4248-aa8f-b97427c8939e","method":"GET","req_id":"4a9d15be-f554-40ee-9c33-4f668cdd85a3","remote_addr":"[fd00:1122:3344:101::d]:65007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":475,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:26:25.146994721Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:39026"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:26:25.197617204Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"725fc22b-a04d-4c71-971c-8a0ecd16e798","remote_addr":"[fd00:1122:3344:101::b]:39026","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50554,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:26:25.197731706Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:46579"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:26:25.238966889Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"888db613-783b-4a4a-88cf-49bed4322070","remote_addr":"[fd00:1122:3344:101::c]:46579","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":41170,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:26:25.239188752Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"96ff7a00-b57a-4548-8067-5dace093cbb9","remote_addr":"[fd00:1122:3344:101::b]:39026","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":104,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:26:25.240991901Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"6275e195-9f8a-4c06-8676-815814735e71","remote_addr":"[fd00:1122:3344:101::c]:46579","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1723,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:26:25.823700433Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:36100"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:26:25.864793603Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"9bf320f4-14d8-4500-8290-ccfbdadbee0a","remote_addr":"[fd00:1122:3344:101::a]:36100","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":41036,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:26:25.865006657Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"989c50d0-e8ef-40d3-8d7b-e8eecc88688c","remote_addr":"[fd00:1122:3344:101::a]:36100","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":90,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:26:52.407911974Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40982"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:26:52.408021666Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"48a3c466-409b-4f00-adfe-7078248b8e12","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40982","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":92,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:26:54.076836776Z","hostname":"sock","pid":1590,"uri":"/metrics/collect/779d9354-af5f-4248-aa8f-b97427c8939e","method":"GET","req_id":"2ff68a0d-3e41-44e5-8945-3b77b5f3a5a6","remote_addr":"[fd00:1122:3344:101::d]:65007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":551,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:26:54.945111776Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:65439"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:26:54.997491708Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"757f91fa-b686-40f1-8001-4238d3ee88ce","remote_addr":"[fd00:1122:3344:101::b]:65439","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":52328,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:26:54.99762133Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:36999"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:26:55.042065724Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"4ac1667b-bad5-4170-b4ba-7ced909c6461","remote_addr":"[fd00:1122:3344:101::c]:36999","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44378,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:26:55.042380889Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"3f58cb22-0d05-4d75-b6c2-611b4c948394","remote_addr":"[fd00:1122:3344:101::b]:65439","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":132,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:26:55.042497901Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"88e24a8a-9811-4806-ba4f-e293aaa07f1e","remote_addr":"[fd00:1122:3344:101::c]:36999","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":198,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:26:55.439571041Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:65350"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:26:55.490512319Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"e6d2662f-2725-4f72-88d2-be9bc2528ef1","remote_addr":"[fd00:1122:3344:101::a]:65350","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50893,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:26:55.490796613Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"68f5c774-348f-49c8-b9a9-e65de0da0c0a","remote_addr":"[fd00:1122:3344:101::a]:65350","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":108,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:03.107869579Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:55012"}
{"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:03.119049328Z","hostname":"sock","pid":1590,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:521","metadata":"InstanceMetadata { silo_id: f8b0c1f9-df2f-4ec0-9bc8-679199022b48, project_id: 083d4960-0f6a-48d0-a5d8-fda21b8d5b7e }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-04-26T00:27:02.630405Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(f8a5ccc0-f44c-4c94-8af4-5625f4311ce7), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-26T00:27:02.654230457Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 0787b510-6e5e-43d2-8077-951992d79e7f, kind: Instance { id: 5c4d1d0c-c50e-4d08-a8c1-5b5774c01542 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 251, 235, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5750498), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0787b510-6e5e-43d2-8077-951992d79e7f, kind: Instance { id: 5c4d1d0c-c50e-4d08-a8c1-5b5774c01542 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 251, 235, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5750498), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0787b510-6e5e-43d2-8077-951992d79e7f, kind: Instance { id: 5c4d1d0c-c50e-4d08-a8c1-5b5774c01542 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 251, 235, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5750498), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(5750498))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0787b510-6e5e-43d2-8077-951992d79e7f, kind: Instance { id: 5c4d1d0c-c50e-4d08-a8c1-5b5774c01542 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 251, 235, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5750498), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-c9567d54fd6\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 7ec0d20c-eea8-480d-a1bd-1951d20fc5c9, read_only_parent: Some(Volume { block_size: 512, id: f01260a4-882b-43a3-9cb5-9419a8377f64, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: a2bd06b5-970b-4f53-a395-1fe729ea96ff, key: Some(\"BB0furI9v8ykv1p/vY5NKZ9NnEREKFO0iluej5C7QEw=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::19]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 7ec0d20c-eea8-480d-a1bd-1951d20fc5c9, key: Some(\"oquhmrRmQiFaflHG7KlkEvWP5y0SaFJu8SMi90kwoLM=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::15]:19000\", \"[fd00:1122:3344:101::19]:19002\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAABrYQOaWJpYAABhA5pYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAa2EDmliaWAAAYQOaWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI1YzRkMWQwYy1jNTBlLTRkMDgtYThjMS01YjU3NzRjMDE1NDIiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSlJFK244dkl0WkpMalRiVTdhckJzZjdSTFBxYVZBMkhjay91cTNrbW5qbCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"f8a5ccc0-f44c-4c94-8af4-5625f4311ce7","instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542"}
{"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:03.11914865Z","hostname":"sock","pid":1590,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:555","instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542"}
{"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:03.119621197Z","hostname":"sock","pid":1590,"component":"InstanceManager","file":"sled-agent/src/instance.rs:938","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 0787b510-6e5e-43d2-8077-951992d79e7f, kind: Instance { id: 5c4d1d0c-c50e-4d08-a8c1-5b5774c01542 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 251, 235, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5750498), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0787b510-6e5e-43d2-8077-951992d79e7f, kind: Instance { id: 5c4d1d0c-c50e-4d08-a8c1-5b5774c01542 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 251, 235, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5750498), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0787b510-6e5e-43d2-8077-951992d79e7f, kind: Instance { id: 5c4d1d0c-c50e-4d08-a8c1-5b5774c01542 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 251, 235, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5750498), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(5750498))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0787b510-6e5e-43d2-8077-951992d79e7f, kind: Instance { id: 5c4d1d0c-c50e-4d08-a8c1-5b5774c01542 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 251, 235, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5750498), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-c9567d54fd6\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 7ec0d20c-eea8-480d-a1bd-1951d20fc5c9, read_only_parent: Some(Volume { block_size: 512, id: f01260a4-882b-43a3-9cb5-9419a8377f64, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: a2bd06b5-970b-4f53-a395-1fe729ea96ff, key: Some(\"BB0furI9v8ykv1p/vY5NKZ9NnEREKFO0iluej5C7QEw=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::19]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 7ec0d20c-eea8-480d-a1bd-1951d20fc5c9, key: Some(\"oquhmrRmQiFaflHG7KlkEvWP5y0SaFJu8SMi90kwoLM=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::15]:19000\", \"[fd00:1122:3344:101::19]:19002\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAABrYQOaWJpYAABhA5pYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAa2EDmliaWAAAYQOaWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI1YzRkMWQwYy1jNTBlLTRkMDgtYThjMS01YjU3NzRjMDE1NDIiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSlJFK244dkl0WkpMalRiVTdhckJzZjdSTFBxYVZBMkhjay91cTNrbW5qbCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(f8a5ccc0-f44c-4c94-8af4-5625f4311ce7), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-26T00:27:02.654230457Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-04-26T00:27:02.630405Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"f8a5ccc0-f44c-4c94-8af4-5625f4311ce7","instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:03.120738755Z","hostname":"sock","pid":1590,"uri":"/instances/5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","method":"PUT","req_id":"e81c3dc5-d2cf-4964-b478-406ba4ebcbc1","remote_addr":"[fd00:1122:3344:101::a]:55012","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":12834,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:03.215703521Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:38513"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:03.217433719Z","hostname":"sock","pid":1590,"uri":"/instances/5c4d1d0c-c50e-4d08-a8c1-5b5774c01542/state","method":"PUT","req_id":"db923101-6bd8-4f2d-a6d2-74e8a31b88d2","remote_addr":"[fd00:1122:3344:101::a]:38513","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1664,"response_code":"200"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:03.239216319Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 251, 235, 59]), slot: 0, vni: Vni { inner: 5750498 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:03.23930046Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:03.272481164Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:03.293942848Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"system/illumos/propolis-server\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlInstance0\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::1:0\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"12400\"/>\n        <propval type=\"astring\" name=\"metric_addr\" value=\"[fd00:1122:3344:101::b]:12221\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:05.21177673Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:05.211968873Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7","instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7\") failed: Property not found. retry in 27.512124ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:27:05.409557168Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7","instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7\") failed: Property not found. retry in 83.864772ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:27:05.444718913Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7","instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7\") failed: Property not found. retry in 279.192283ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:27:05.544582417Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7","instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7\") failed: Property not found. retry in 325.348674ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:27:05.84104041Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7","instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7\") failed: Property not found. retry in 785.341289ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:27:06.179596599Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7","instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7\") failed: Property not found. retry in 591.029567ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:27:06.975238982Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7","instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7\") failed: Property not found. retry in 964.209746ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:27:07.576227037Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7","instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7\") failed: Property not found. retry in 1.327044291s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:27:08.550627232Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7","instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7\") failed: Property not found. retry in 1.013497604s","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:27:09.887127204Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7","instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Started propolis in zone: oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:10.92884063Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"sled-agent/src/instance.rs:1423"}
{"msg":"wait for service Some(\"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7\") failed: Property not found. retry in 47.730458ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:27:10.937815004Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7\") failed: Property not found. retry in 94.326731ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:27:10.995287008Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7\") failed: Property not found. retry in 177.630283ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:27:11.098792231Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7\") failed: Property not found. retry in 253.434162ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:27:11.287164127Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7\") failed: Property not found. retry in 414.603531ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:27:11.551652377Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:11.977683041Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"sled-agent/src/instance.rs:1432"}
{"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:12.050188796Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"sled-agent/src/instance.rs:1446"}
{"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAABrYQOaWJpYAABhA5pYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAa2EDmliaWAAAYQOaWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI1YzRkMWQwYy1jNTBlLTRkMDgtYThjMS01YjU3NzRjMDE1NDIiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSlJFK244dkl0WkpMalRiVTdhckJzZjdSTFBxYVZBMkhjay91cTNrbW5qbCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-c9567d54fd6\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 7ec0d20c-eea8-480d-a1bd-1951d20fc5c9, read_only_parent: Some(Volume { block_size: 512, id: f01260a4-882b-43a3-9cb5-9419a8377f64, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: a2bd06b5-970b-4f53-a395-1fe729ea96ff, key: Some(\"BB0furI9v8ykv1p/vY5NKZ9NnEREKFO0iluej5C7QEw=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::19]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 7ec0d20c-eea8-480d-a1bd-1951d20fc5c9, key: Some(\"oquhmrRmQiFaflHG7KlkEvWP5y0SaFJu8SMi90kwoLM=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::15]:19000\", \"[fd00:1122:3344:101::19]:19002\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 5c4d1d0c-c50e-4d08-a8c1-5b5774c01542, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: 083d4960-0f6a-48d0-a5d8-fda21b8d5b7e, silo_id: f8b0c1f9-df2f-4ec0-9bc8-679199022b48 }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:12.052565724Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"sled-agent/src/instance.rs:685"}
{"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:12.138085198Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"sled-agent/src/instance.rs:687"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-04-26T00:27:12.139132735Z }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:12.139155545Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:12.139318558Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-04-26T00:27:12.139132735Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(f8a5ccc0-f44c-4c94-8af4-5625f4311ce7), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-26T00:27:02.654230457Z }","propolis_id":"f8a5ccc0-f44c-4c94-8af4-5625f4311ce7"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:12.139362198Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(f8a5ccc0-f44c-4c94-8af4-5625f4311ce7), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-26T00:27:02.654230457Z }, propolis_id: f8a5ccc0-f44c-4c94-8af4-5625f4311ce7, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-04-26T00:27:12.139132735Z } }","instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-04-26T00:27:12.242408864Z }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:12.242440964Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:12.242469915Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-04-26T00:27:12.242408864Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(f8a5ccc0-f44c-4c94-8af4-5625f4311ce7), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-26T00:27:02.654230457Z }","propolis_id":"f8a5ccc0-f44c-4c94-8af4-5625f4311ce7"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:12.242481135Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(f8a5ccc0-f44c-4c94-8af4-5625f4311ce7), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-26T00:27:02.654230457Z }, propolis_id: f8a5ccc0-f44c-4c94-8af4-5625f4311ce7, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-04-26T00:27:12.242408864Z } }","instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542"}
{"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:12.397244342Z","hostname":"sock","pid":1590,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:300"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:12.397272082Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:12.438576736Z","hostname":"sock","pid":1590,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.99999582s","next_cleanup":"Instant { tv_sec: 2114, tv_nsec: 94639263 }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:22.439824966Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44099"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:22.439922577Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"b055ae00-13b1-498f-b468-75bca4c6f18d","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44099","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":88,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:24.076554271Z","hostname":"sock","pid":1590,"uri":"/metrics/collect/779d9354-af5f-4248-aa8f-b97427c8939e","method":"GET","req_id":"a8c38238-c3e0-436a-a6f0-b98b2ec32f05","remote_addr":"[fd00:1122:3344:101::d]:65007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":541,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:25.041077217Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:42443"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:25.093716203Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"4d51ae30-390a-4706-b011-228647e54f81","remote_addr":"[fd00:1122:3344:101::b]:42443","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":52566,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:25.094050498Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"742ae40b-a12d-49da-a3c5-cc92923e9fcf","remote_addr":"[fd00:1122:3344:101::b]:42443","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":101,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:25.167991586Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:34092"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:25.217506621Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"1322b228-b27f-4ef1-98b9-ffeb621ec25d","remote_addr":"[fd00:1122:3344:101::c]:34092","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49413,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:25.217770156Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"c43fbd40-5e63-45ed-9989-c3aff04b5ab1","remote_addr":"[fd00:1122:3344:101::c]:34092","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":102,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:25.523487968Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:46719"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:25.569982635Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"3958b828-d453-4670-853d-59434a8b9b12","remote_addr":"[fd00:1122:3344:101::a]:46719","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46370,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:25.570235339Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"01834656-4678-4c6e-9c51-1ef2e107997f","remote_addr":"[fd00:1122:3344:101::a]:46719","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":97,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:25.878696644Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:51300"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:25.880216248Z","hostname":"sock","pid":1590,"uri":"/instances/5c4d1d0c-c50e-4d08-a8c1-5b5774c01542/state","method":"PUT","req_id":"aa95ca85-5922-4656-bde4-e7d4a371d31b","remote_addr":"[fd00:1122:3344:101::a]:51300","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1436,"response_code":"200"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-04-26T00:27:25.880311590Z }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:25.88033999Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:25.88035727Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-04-26T00:27:25.880311590Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(f8a5ccc0-f44c-4c94-8af4-5625f4311ce7), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-26T00:27:02.654230457Z }","propolis_id":"f8a5ccc0-f44c-4c94-8af4-5625f4311ce7"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:25.880394351Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(f8a5ccc0-f44c-4c94-8af4-5625f4311ce7), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-26T00:27:02.654230457Z }, propolis_id: f8a5ccc0-f44c-4c94-8af4-5625f4311ce7, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-04-26T00:27:25.880311590Z } }","instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-04-26T00:27:25.913179898Z }","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:25.913208388Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:25.913238739Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-04-26T00:27:25.913179898Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-04-26T00:27:25.913179898Z }","propolis_id":"f8a5ccc0-f44c-4c94-8af4-5625f4311ce7"}
{"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:25.913272459Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:25.913279289Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:25.913460662Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:25.913530123Z","hostname":"sock","pid":1590,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:437","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/47941014-d07d-4a89-a7ad-e8307b721c42/debug/bundle/zone\", \"/pool/int/c162d7a8-2ba1-425d-83b7-9a5b15f2af8a/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/d264dd09-dd1e-4b44-ac20-70d4e8728824/crypt/debug\", \"/pool/ext/1962e339-1294-4041-a983-29e64bdb2ffe/crypt/debug\", \"/pool/ext/29f0e314-9945-472b-979d-b91f7fcb5170/crypt/debug\", \"/pool/ext/b4f1d1aa-0185-4f19-9f65-a972831259c3/crypt/debug\", \"/pool/ext/fa79b281-89d1-42f4-b871-62a5aaaf9922/crypt/debug\", \"/pool/ext/42895196-0869-4ea3-9c4e-285bbcea1a80/crypt/debug\", \"/pool/ext/0de19191-ee35-4d6e-ab04-5e684dafd16c/crypt/debug\", \"/pool/ext/1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f/crypt/debug\", \"/pool/ext/73bca2cf-2bf8-443a-bf93-681a0fe6b326/crypt/debug\"] }","zone_name":"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7"}
{"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:26.57927187Z","hostname":"sock","pid":1590,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1230","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7\", bundle_id: 470544f7-ea74-448a-8641-aca8e69e2530 }, time_created: 2024-04-26T00:27:25.913797078Z, version: 0, cause: TerminatedInstance }"}
{"msg":"Halting and removing zone: oxz_propolis-server_f8a5ccc0-f44c-4c94-8af4-5625f4311ce7","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:27:26.579368151Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:28.360889393Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:28.387695964Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-04-26T00:27:25.913179898Z }, propolis_id: f8a5ccc0-f44c-4c94-8af4-5625f4311ce7, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-04-26T00:27:28.387626042Z } }","instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:28.4851941Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:52107"}
{"msg":"unset_virtual_nic_host unimplmented","v":0,"name":"SledAgent","level":40,"time":"2024-04-26T00:27:28.486335818Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:620"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:28.486347429Z","hostname":"sock","pid":1590,"uri":"/v2p/0787b510-6e5e-43d2-8077-951992d79e7f","method":"DELETE","req_id":"8f01115f-2683-4be7-94ef-ad29a6207f82","remote_addr":"[fd00:1122:3344:101::a]:52107","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1036,"response_code":"204"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:28.805360463Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-04-26T00:27:25.913179898Z }, propolis_id: f8a5ccc0-f44c-4c94-8af4-5625f4311ce7, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-04-26T00:27:28.387626042Z } }","instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542"}
{"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-04-26T00:27:28.808039166Z","hostname":"sock","pid":1590,"instance_id":"5c4d1d0c-c50e-4d08-a8c1-5b5774c01542","component":"InstanceManager","file":"sled-agent/src/instance.rs:723"}