[ Mar 30 05:42:13 Enabled. ]
[ Mar 30 05:42:13 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ]
[ Mar 30 05:42:13 Method "start" exited with status 0. ]
[ Mar 30 05:42:13 Rereading configuration. ]
[ Mar 30 05:42:13 No 'refresh' method defined.  Treating as :true. ]
note: configured to log to "/dev/stdout"
{"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:13.723968371Z","hostname":"buskin","pid":1618,"file":"illumos-utils/src/running_zone.rs:163"}
{"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:13.724120293Z","hostname":"buskin","pid":1618,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:253"}
{"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:13.777854266Z","hostname":"buskin","pid":1618,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:260","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"}
{"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:13.777896427Z","hostname":"buskin","pid":1618,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"}
{"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:13.893750354Z","hostname":"buskin","pid":1618,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"}
{"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:13.902826338Z","hostname":"buskin","pid":1618,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"}
{"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:13.910437536Z","hostname":"buskin","pid":1618,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:270","path":"/var/run/oxide/"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.514202254Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:9fb2::, len: 64 }"}
{"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.654929011Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:115"}
{"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.654979442Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:127"}
{"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.655046863Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:137"}
{"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.655133395Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:157","sled_mode":"Scrimlet { asic: SoftNpuZone }"}
{"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.655142135Z","hostname":"buskin","pid":1618,"component":"DumpSetup-archival","file":"sled-agent/src/dump_setup.rs:291"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.655153115Z","hostname":"buskin","pid":1618,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"}
{"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.655173535Z","hostname":"buskin","pid":1618,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:609"}
{"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.658031067Z","hostname":"buskin","pid":1618,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"}
{"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:14.658043687Z","hostname":"buskin","pid":1618,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:658"}
{"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.658051627Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:171"}
{"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.658056298Z","hostname":"buskin","pid":1618,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:716"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.658074418Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/m2_0.vdev"}
{"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.658136309Z","hostname":"buskin","pid":1618,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:261","underlay_network_info":"None"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.65821361Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.658235121Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.658252731Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.669715499Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"}
{"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.669765219Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:312","id":"eaeb58f9-7756-4551-8a4b-dc00787e5906"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.768805242Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: eaeb58f9-7756-4551-8a4b-dc00787e5906, kind: Internal }"}
{"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.971308577Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.971345308Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/m2_1.vdev"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.971350508Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.971361528Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.971365678Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.971374669Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.979829302Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"}
{"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:14.979859862Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:312","id":"f076b946-15f4-4270-9a34-aa6e75797fbb"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.066139094Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: f076b946-15f4-4270-9a34-aa6e75797fbb, kind: Internal }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269485534Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_0.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269526845Z","hostname":"buskin","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: eaeb58f9-7756-4551-8a4b-dc00787e5906, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: f076b946-15f4-4270-9a34-aa6e75797fbb, kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269559746Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269576746Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269584196Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_1.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269590056Z","hostname":"buskin","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: eaeb58f9-7756-4551-8a4b-dc00787e5906, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: f076b946-15f4-4270-9a34-aa6e75797fbb, kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269646327Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269652497Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269690118Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_2.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269709758Z","hostname":"buskin","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: eaeb58f9-7756-4551-8a4b-dc00787e5906, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: f076b946-15f4-4270-9a34-aa6e75797fbb, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269731079Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269748599Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.26977961Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_3.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.26979607Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.26980829Z","hostname":"buskin","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: eaeb58f9-7756-4551-8a4b-dc00787e5906, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: f076b946-15f4-4270-9a34-aa6e75797fbb, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269825971Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269831621Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_4.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269853891Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269858661Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269863841Z","hostname":"buskin","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: eaeb58f9-7756-4551-8a4b-dc00787e5906, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: f076b946-15f4-4270-9a34-aa6e75797fbb, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269884252Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_5.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269903622Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269907892Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269923482Z","hostname":"buskin","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: eaeb58f9-7756-4551-8a4b-dc00787e5906, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: f076b946-15f4-4270-9a34-aa6e75797fbb, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269943403Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_6.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269949083Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269953223Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269959993Z","hostname":"buskin","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: eaeb58f9-7756-4551-8a4b-dc00787e5906, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: f076b946-15f4-4270-9a34-aa6e75797fbb, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269980213Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_7.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.269998674Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.270003704Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.270018864Z","hostname":"buskin","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: eaeb58f9-7756-4551-8a4b-dc00787e5906, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: f076b946-15f4-4270-9a34-aa6e75797fbb, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.270040895Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_8.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.270046044Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.270051575Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.270062165Z","hostname":"buskin","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: eaeb58f9-7756-4551-8a4b-dc00787e5906, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: f076b946-15f4-4270-9a34-aa6e75797fbb, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.270093195Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:87"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.270123576Z","hostname":"buskin","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: eaeb58f9-7756-4551-8a4b-dc00787e5906, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: f076b946-15f4-4270-9a34-aa6e75797fbb, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.270157386Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:89"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.270162127Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.270171877Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:195"}
{"msg":"No ledger in /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.270259548Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.270283449Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.270293589Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"}
{"msg":"No ledger in /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.27031448Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.27033038Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.27033547Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"}
{"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.270376681Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:200"}
{"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.270406941Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:215"}
{"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.281589173Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:701"}
{"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.281659985Z","hostname":"buskin","pid":1618,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999919109s","next_cleanup":"Instant { tv_sec: 1569, tv_nsec: 716438591 }"}
{"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.281682145Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3588"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.281689935Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.281746196Z","hostname":"buskin","pid":1618,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.283740383Z","hostname":"buskin","pid":1618,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.284785591Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"}
{"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.284882573Z","hostname":"buskin","pid":1618,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"}
{"msg":"Injecting RSS configuration: RackInitializeRequest {\n    trust_quorum_peers: None,\n    bootstrap_discovery: OnlyOurs,\n    ntp_servers: [\n        \"0.pool.ntp.org\",\n    ],\n    dns_servers: [\n        1.1.1.1,\n        9.9.9.9,\n    ],\n    internal_services_ip_pool_ranges: [\n        V4(\n            Ipv4Range {\n                first: 10.151.1.164,\n                last: 10.151.1.173,\n            },\n        ),\n    ],\n    external_dns_ips: [\n        10.151.1.164,\n        10.151.1.165,\n    ],\n    external_dns_zone_name: \"oxide.test\",\n    external_certificates: \"<redacted>\",\n    recovery_silo: RecoverySiloConfig {\n        silo_name: Name(\n            \"recovery\",\n        ),\n        user_name: UserId(\n            \"recovery\",\n        ),\n        user_password_hash: NewPasswordHash(\n            PasswordHashString {\n                string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n                encoding: B64,\n            },\n        ),\n    },\n    rack_network_config: RackNetworkConfigV1 {\n        rack_subnet: Ipv6Network {\n            addr: fd00:1122:3344:100::,\n            prefix: 56,\n        },\n        infra_ip_first: 10.151.1.174,\n        infra_ip_last: 10.151.1.174,\n        ports: [\n            PortConfigV1 {\n                routes: [\n                    RouteConfig {\n                        destination: V4(\n                            Ipv4Network {\n                                addr: 0.0.0.0,\n                                prefix: 0,\n                            },\n                        ),\n                        nexthop: 10.151.1.100,\n                    },\n                ],\n                addresses: [\n                    V4(\n                        Ipv4Network {\n                            addr: 10.151.1.174,\n                            prefix: 24,\n                        },\n                    ),\n                ],\n                switch: Switch0,\n                port: \"qsfp0\",\n                uplink_port_speed: Speed40G,\n                uplink_port_fec: None,\n                bgp_peers: [],\n                autoneg: false,\n            },\n        ],\n        bgp: [],\n        bfd: [],\n    },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.285178929Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:997"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.285186919Z","hostname":"buskin","pid":1618,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.28524873Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.28528822Z","hostname":"buskin","pid":1618,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.285303891Z","hostname":"buskin","pid":1618,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.285310661Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1057"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.285340921Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.285348941Z","hostname":"buskin","pid":1618,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.285362542Z","hostname":"buskin","pid":1618,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.285374832Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1098"}
{"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.285379412Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"}
{"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.285383692Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.285405533Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Writing ledger to /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.285417263Z","hostname":"buskin","pid":1618,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.285738899Z","hostname":"buskin","pid":1618,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.285885121Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"}
{"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.285898951Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1131"}
{"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.285951543Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"}
{"msg":"Writing ledger to /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.285959792Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.286122035Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.286275168Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"}
{"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.28634355Z","hostname":"buskin","pid":1618,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 93b0cc91-dc62-481c-9ea1-425d29f058ae, rack_id: 98531190-dbea-48cf-b76b-c50c819faca1, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }"}
{"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.286459872Z","hostname":"buskin","pid":1618,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:57598","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"}
{"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 93b0cc91-dc62-481c-9ea1-425d29f058ae, rack_id: 98531190-dbea-48cf-b76b-c50c819faca1, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.286563974Z","hostname":"buskin","pid":1618,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:356"}
{"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.286570854Z","hostname":"buskin","pid":1618,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:371"}
{"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.286593294Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/server.rs:44"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.286599914Z","hostname":"buskin","pid":1618,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.286622784Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.286627485Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:530"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.286631285Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"}
{"msg":"No ledger in /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.286671916Z","hostname":"buskin","pid":1618,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.286678025Z","hostname":"buskin","pid":1618,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.286684816Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:519"}
{"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.286689946Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:547"}
{"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.289405225Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3819"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.298224574Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.308399549Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.317587525Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.326249222Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.334550132Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.343663837Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.352060899Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.360598803Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.369185699Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.36924537Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:558"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.36925079Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.36925605Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:370"}
{"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.36926123Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"}
{"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.3692665Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:449","result":"DisksManagementResult { status: [] }"}
{"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.378191862Z","hostname":"buskin","pid":1618,"sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:355"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.378298064Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.378306824Z","hostname":"buskin","pid":1618,"sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:367"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.378387556Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 64.415612ms","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.381607674Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c032164d-abdb-41e6-a4b2-78d6ec53ba55\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:42:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c032164d-abdb-41e6-a4b2-78d6ec53ba55\" }"}
{"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.38966727Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"attempting to create encrypted zvol: name=\"oxi_eaeb58f9-7756-4551-8a4b-dc00787e5906/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.392338388Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/swap_device.rs:153"}
{"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.407698206Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"successfully created encrypted zvol: name=\"oxi_eaeb58f9-7756-4551-8a4b-dc00787e5906/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.4156628Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/swap_device.rs:249"}
{"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_eaeb58f9-7756-4551-8a4b-dc00787e5906/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.415708231Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/swap_device.rs:96"}
{"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.420531058Z","hostname":"buskin","pid":1618,"sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:382"}
{"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.420540819Z","hostname":"buskin","pid":1618,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"}
{"msg":"Ensuring dataset oxi_eaeb58f9-7756-4551-8a4b-dc00787e5906/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.420545489Z","hostname":"buskin","pid":1618,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.446925986Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 152.677356ms","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.447496227Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7966b747-4356-423c-80c7-7b3ec958e8d4\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:42:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7966b747-4356-423c-80c7-7b3ec958e8d4\" }"}
{"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.487592072Z","hostname":"buskin","pid":1618,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.60072116Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 190.739895ms","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.6012644Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"176f5c2a-e672-4aa7-b646-3bd6c0c1fc69\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:42:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"176f5c2a-e672-4aa7-b646-3bd6c0c1fc69\" }"}
{"msg":"Mounting oxi_eaeb58f9-7756-4551-8a4b-dc00787e5906/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.705840003Z","hostname":"buskin","pid":1618,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"}
{"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.718709035Z","hostname":"buskin","pid":1618,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"}
{"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.718797107Z","hostname":"buskin","pid":1618,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"}
{"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.718822867Z","hostname":"buskin","pid":1618,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"}
{"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.718851758Z","hostname":"buskin","pid":1618,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.792844147Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 89.239827ms","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.793374187Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3319cbd7-7995-457b-b0a7-e04da51e93fa\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:42:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3319cbd7-7995-457b-b0a7-e04da51e93fa\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.883419467Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 230.871811ms","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:15.883816124Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e2c41dfb-1832-41d4-bd41-53da445c17e4\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:42:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e2c41dfb-1832-41d4-bd41-53da445c17e4\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.115472887Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 218.346477ms","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.116058577Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"60db722d-60c7-42c8-9b69-294bb2217509\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:42:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"60db722d-60c7-42c8-9b69-294bb2217509\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.336159151Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 347.876502ms","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.336871594Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d7f0c78e-3a39-4abb-94dc-b2f2992ff602\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:42:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d7f0c78e-3a39-4abb-94dc-b2f2992ff602\" }"}
{"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.43000639Z","hostname":"buskin","pid":1618,"sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"}
{"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.476844768Z","hostname":"buskin","pid":1618,"component":"VmmReservoirManager","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:266"}
{"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.476892828Z","hostname":"buskin","pid":1618,"component":"VmmReservoirManager","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:274"}
{"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.47696156Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:983","underlay_address":"fd00:1122:3344:101::1"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.47697015Z","hostname":"buskin","pid":1618,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.477982608Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"}
{"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.478015009Z","hostname":"buskin","pid":1618,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:426"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.478040499Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Writing ledger to /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.47805799Z","hostname":"buskin","pid":1618,"component":"SledAgentMain","file":"common/src/ledger.rs:199"}
{"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:16.478219743Z","hostname":"buskin","pid":1618,"sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1178","retry_after":"332.480585ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""}
{"msg":"Writing ledger to /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.478301684Z","hostname":"buskin","pid":1618,"component":"SledAgentMain","file":"common/src/ledger.rs:199"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:16.478352755Z","hostname":"buskin","pid":1618,"component":"NexusNotifierTask","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.478486057Z","hostname":"buskin","pid":1618,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"}
{"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.478505558Z","hostname":"buskin","pid":1618,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:261","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.1.100 }], addresses: [V4(Ipv4Network { addr: 10.151.1.174, prefix: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"}
{"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.478513808Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3588"}
{"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.478518268Z","hostname":"buskin","pid":1618,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:57598","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"}
{"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.478602759Z","hostname":"buskin","pid":1618,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"}
{"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.478608319Z","hostname":"buskin","pid":1618,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.678398046Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.678503068Z","hostname":"buskin","pid":1618,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.678568189Z","hostname":"buskin","pid":1618,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.678579279Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.678594309Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.685424383Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.710733461Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:60256"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.710913144Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.711008986Z","hostname":"buskin","pid":1618,"uri":"/inventory","method":"GET","req_id":"afafbef2-3118-4fd5-8e0b-9af682093dbe","remote_addr":"[fd00:1122:3344:101::1]:60256","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":171,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.741506298Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:55643"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.74162328Z","hostname":"buskin","pid":1618,"uri":"/sled-role","method":"GET","req_id":"f62e8fe6-3b21-4b19-9af3-691b6f263288","remote_addr":"[fd00:1122:3344:101::1]:55643","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":41,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.743062146Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Writing ledger to /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.743074186Z","hostname":"buskin","pid":1618,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.743399992Z","hostname":"buskin","pid":1618,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.743602326Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:802"}
{"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: bd856e1e-ac7c-42f0-8254-c13876319841, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, pool_id: 82347231-68ab-4934-8007-b28d9a3a3777 }, OmicronPhysicalDiskConfig { id: e284228c-0e5b-4599-8158-4449d9538b60, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, pool_id: 28899eb6-914e-44b6-b77e-9d82d92186d6 }, OmicronPhysicalDiskConfig { id: 447cadf7-31c9-49a1-9141-b94373722b3b, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, pool_id: 6de8542c-123a-4abc-9a88-e72f1063a9f4 }, OmicronPhysicalDiskConfig { id: d0107f86-d382-463c-a450-b342158c84f7, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, pool_id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5 }, OmicronPhysicalDiskConfig { id: 4d836df1-a884-4e12-adcd-7afb4d0e2299, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, pool_id: 5c3019ca-328c-47ab-9982-06e092199db8 }, OmicronPhysicalDiskConfig { id: be27dfe8-0d06-46ec-abb5-1646d8adc826, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, pool_id: 3c92b6bd-8180-45a3-9123-a99ddc649159 }, OmicronPhysicalDiskConfig { id: 47eb38d4-25bb-4df1-94a2-9e15e1e2f8f0, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, pool_id: ea19d3d4-e058-46fb-981b-9ccb255f563c }, OmicronPhysicalDiskConfig { id: a050ef5b-d930-47c7-9e01-49fe59c8434a, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, pool_id: e22df0ad-bbf8-4902-93d6-e67ff23c3234 }, OmicronPhysicalDiskConfig { id: 366f48ff-1929-4f47-a136-197d43fc6605, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, pool_id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.781743766Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:344"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.781835448Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:58072"}
{"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, id: bd856e1e-ac7c-42f0-8254-c13876319841, pool_id: 82347231-68ab-4934-8007-b28d9a3a3777 }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, id: e284228c-0e5b-4599-8158-4449d9538b60, pool_id: 28899eb6-914e-44b6-b77e-9d82d92186d6 }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, id: 447cadf7-31c9-49a1-9141-b94373722b3b, pool_id: 6de8542c-123a-4abc-9a88-e72f1063a9f4 }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, id: d0107f86-d382-463c-a450-b342158c84f7, pool_id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5 }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, id: 4d836df1-a884-4e12-adcd-7afb4d0e2299, pool_id: 5c3019ca-328c-47ab-9982-06e092199db8 }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, id: be27dfe8-0d06-46ec-abb5-1646d8adc826, pool_id: 3c92b6bd-8180-45a3-9123-a99ddc649159 }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, id: 47eb38d4-25bb-4df1-94a2-9e15e1e2f8f0, pool_id: ea19d3d4-e058-46fb-981b-9ccb255f563c }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, id: a050ef5b-d930-47c7-9e01-49fe59c8434a, pool_id: e22df0ad-bbf8-4902-93d6-e67ff23c3234 }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, id: 366f48ff-1929-4f47-a136-197d43fc6605, pool_id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.782061952Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.782067662Z","hostname":"buskin","pid":1618,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.782073132Z","hostname":"buskin","pid":1618,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.782078072Z","hostname":"buskin","pid":1618,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:681"}
{"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.782082632Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.782087452Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.782093123Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.782097533Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:16.882396188Z","hostname":"buskin","pid":1618,"sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1178","retry_after":"712.795655ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.889322953Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:16.889337414Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"82347231-68ab-4934-8007-b28d9a3a3777"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T05:42:17.579128959Z","hostname":"buskin","pid":1618,"component":"ProbeManager","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:17.596609825Z","hostname":"buskin","pid":1618,"sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1178","retry_after":"1.121519062s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:17.999805773Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:18.192634003Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:18.192763136Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:18.192777726Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:18.192802237Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:18.192815207Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:18.19297695Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:18.19298723Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T05:42:18.653416624Z","hostname":"buskin","pid":1618,"component":"ProbeManager","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:18.653657358Z","hostname":"buskin","pid":1618,"component":"NexusNotifierTask","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:18.852964255Z","hostname":"buskin","pid":1618,"sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1178","retry_after":"2.092164439s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:18.853700099Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"}
{"msg":"Automatically destroying dataset: oxp_82347231-68ab-4934-8007-b28d9a3a3777/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:19.085107276Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:19.276417109Z","hostname":"buskin","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 34.760811ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:20.125761552Z","hostname":"buskin","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T05:42:20.126871582Z","hostname":"buskin","pid":1618,"component":"ProbeManager","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 90.579483ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:20.76872449Z","hostname":"buskin","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:20.769985543Z","hostname":"buskin","pid":1618,"component":"NexusNotifierTask","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 208.44782ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:21.442867032Z","hostname":"buskin","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T05:42:21.445616091Z","hostname":"buskin","pid":1618,"component":"ProbeManager","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:21.445672602Z","hostname":"buskin","pid":1618,"sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1178","retry_after":"4.400028945s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 579.942719ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:21.846006448Z","hostname":"buskin","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Looking for unencrypted datasets in oxp_82347231-68ab-4934-8007-b28d9a3a3777","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:22.276340657Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:22.697699063Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:22.697812955Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:22.697827386Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:22.697841336Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:22.697848296Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T05:42:22.905138018Z","hostname":"buskin","pid":1618,"component":"ProbeManager","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:22.905186479Z","hostname":"buskin","pid":1618,"component":"NexusNotifierTask","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.118681489s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:22.905321651Z","hostname":"buskin","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:22.911623355Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:22.911633766Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"28899eb6-914e-44b6-b77e-9d82d92186d6"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T05:42:23.978299761Z","hostname":"buskin","pid":1618,"component":"ProbeManager","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:23.989042206Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:24.427639464Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:24.427745756Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:24.427802867Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:24.427862888Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:24.427900589Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:24.429542069Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:24.429558689Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.358620191s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:24.650842054Z","hostname":"buskin","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:24.906614594Z","hostname":"buskin","pid":1618,"component":"NexusNotifierTask","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T05:42:25.13131025Z","hostname":"buskin","pid":1618,"component":"ProbeManager","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:25.133316277Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"}
{"msg":"Automatically destroying dataset: oxp_28899eb6-914e-44b6-b77e-9d82d92186d6/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:25.377644329Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T05:42:26.501947719Z","hostname":"buskin","pid":1618,"component":"ProbeManager","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:26.50198855Z","hostname":"buskin","pid":1618,"sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1178","retry_after":"10.589991045s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""}
{"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:26.941634127Z","hostname":"buskin","pid":1618,"component":"VmmReservoirManager","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:278"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:26.943811957Z","hostname":"buskin","pid":1618,"component":"NexusNotifierTask","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Looking for unencrypted datasets in oxp_28899eb6-914e-44b6-b77e-9d82d92186d6","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.024564928Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.033597511Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.033662862Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.033674872Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.033685322Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.033691103Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.042986751Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.043042182Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"6de8542c-123a-4abc-9a88-e72f1063a9f4"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.136354211Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.143931648Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.143987919Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.143999549Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.14402679Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.14404659Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.144192633Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.144205163Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.178085426Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"}
{"msg":"Automatically destroying dataset: oxp_6de8542c-123a-4abc-9a88-e72f1063a9f4/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.186445438Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_6de8542c-123a-4abc-9a88-e72f1063a9f4","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.309182169Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.317228495Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.317285086Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.317293996Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.317301016Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.317307036Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.327402009Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.327430999Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"e43353e3-7e66-4e6f-bd56-19f8143ee8b5"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.425316131Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.433987608Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.434013959Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.434020259Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.434039769Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.434046919Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.434181962Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.434187512Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.470165243Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"}
{"msg":"Automatically destroying dataset: oxp_e43353e3-7e66-4e6f-bd56-19f8143ee8b5/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.478666597Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T05:42:27.505099945Z","hostname":"buskin","pid":1618,"component":"ProbeManager","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Looking for unencrypted datasets in oxp_e43353e3-7e66-4e6f-bd56-19f8143ee8b5","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.602483228Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.610642925Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.610700797Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.610711617Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.610720347Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.610734197Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.620337841Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.620381872Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"5c3019ca-328c-47ab-9982-06e092199db8"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.71419566Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.722300897Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.722378538Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.722392748Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.722455149Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.72247786Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.722655033Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.722663743Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.75785455Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"}
{"msg":"Automatically destroying dataset: oxp_5c3019ca-328c-47ab-9982-06e092199db8/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.765718143Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_5c3019ca-328c-47ab-9982-06e092199db8","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.889400181Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.897457417Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.897505508Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.897516368Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.897527078Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.897550959Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.906938599Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:27.906972029Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"3c92b6bd-8180-45a3-9123-a99ddc649159"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.000876659Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.008812362Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.008863643Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.008880193Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.008933944Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.008953395Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.009103197Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.009115368Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.046322501Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"}
{"msg":"Automatically destroying dataset: oxp_3c92b6bd-8180-45a3-9123-a99ddc649159/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.054260205Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_3c92b6bd-8180-45a3-9123-a99ddc649159","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.175817975Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.183589436Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.183632307Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.183640087Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.183665527Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.183672437Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.192792752Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.192830383Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"ea19d3d4-e058-46fb-981b-9ccb255f563c"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.287873283Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.295500141Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.295555722Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.295570373Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.295703795Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.295724675Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.295842867Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.295855808Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.331567284Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"}
{"msg":"Automatically destroying dataset: oxp_ea19d3d4-e058-46fb-981b-9ccb255f563c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.339366525Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_ea19d3d4-e058-46fb-981b-9ccb255f563c","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.459491649Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.467289801Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.467338811Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.467350062Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.467357162Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.467367032Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.476855994Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.476895915Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"e22df0ad-bbf8-4902-93d6-e67ff23c3234"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.570182983Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.577792981Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.577840622Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.577850522Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.577903383Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.577922193Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.578092876Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.578109027Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T05:42:28.582482446Z","hostname":"buskin","pid":1618,"component":"ProbeManager","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.614115028Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"}
{"msg":"Automatically destroying dataset: oxp_e22df0ad-bbf8-4902-93d6-e67ff23c3234/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.62193966Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_e22df0ad-bbf8-4902-93d6-e67ff23c3234","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.743825856Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.75178071Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.751834791Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.751844071Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.751852131Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.751859312Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.761898343Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.761936914Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"c7e0e489-6c2b-472e-8a1f-efe83bb548dd"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.858670325Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.866453906Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.866491916Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.866500547Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.866528877Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.866540367Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.86667405Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.86668255Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.903673439Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"}
{"msg":"Automatically destroying dataset: oxp_c7e0e489-6c2b-472e-8a1f-efe83bb548dd/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.911556032Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:28.945336193Z","hostname":"buskin","pid":1618,"component":"NexusNotifierTask","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:28.967744978Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2286"}
{"msg":"Looking for unencrypted datasets in oxp_c7e0e489-6c2b-472e-8a1f-efe83bb548dd","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.0341727Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.041185847Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.041243768Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Writing ledger to /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.041278099Z","hostname":"buskin","pid":1618,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.041486053Z","hostname":"buskin","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: eaeb58f9-7756-4551-8a4b-dc00787e5906, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: f076b946-15f4-4270-9a34-aa6e75797fbb, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Writing ledger to /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.041657156Z","hostname":"buskin","pid":1618,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.041950451Z","hostname":"buskin","pid":1618,"uri":"/omicron-physical-disks","method":"PUT","req_id":"31715b1b-3c72-42f2-87a9-5aaa0cf97fd5","remote_addr":"[fd00:1122:3344:101::1]:58072","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":12260047,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.072740028Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.072985103Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:46225"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.073271458Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.073280978Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.123159411Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.123197102Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.172621446Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.172658517Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.222429238Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.222485609Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.222512839Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:188"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.222577371Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.267896721Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.267957742Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.267965212Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.315979131Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.319492345Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.323173961Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.334861893Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.339534328Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:29.343372607Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T05:42:29.585253215Z","hostname":"buskin","pid":1618,"component":"ProbeManager","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T05:42:30.586678071Z","hostname":"buskin","pid":1618,"component":"ProbeManager","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:30.944981536Z","hostname":"buskin","pid":1618,"component":"NexusNotifierTask","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Profile for oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService2\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:2::2\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:2::1\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:2::1]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"[fd00:1122:3344:2::1]:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:31.700894938Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:31.700957419Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:31.701173273Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService3\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:3::2\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:3::1\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:3::1]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"[fd00:1122:3344:3::1]:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:32.185431008Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:32.185506989Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"}
{"msg":"Profile for oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService1\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:1::2\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:1::1\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:1::1]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"[fd00:1122:3344:1::1]:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:32.667454913Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"wait for service Some(\"oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490\") failed: Property not found. retry in 40.811764ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:32.67501562Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:32.675079611Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:32.678949381Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:32.679013622Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490\") failed: Property not found. retry in 63.858422ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:32.725607775Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490\") failed: Property not found. retry in 140.797668ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:32.799186217Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795\") failed: Property not found. retry in 58.25315ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:32.805561083Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2\") failed: Property not found. retry in 61.420802ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:32.812199372Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795\") failed: Property not found. retry in 131.022896ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:32.949398396Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490\") failed: Property not found. retry in 273.851454ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:32.991144001Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2\") failed: Property not found. retry in 63.50206ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:33.02808175Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795\") failed: Property not found. retry in 267.54727ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:33.116042832Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2\") failed: Property not found. retry in 212.993567ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:33.124471375Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490\") failed: Property not found. retry in 518.237125ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:33.275901945Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2\") failed: Property not found. retry in 455.712682ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:33.350015137Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795\") failed: Property not found. retry in 383.307162ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:33.382549436Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795\") failed: Property not found. retry in 507.540896ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:33.775545619Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490\") failed: Property not found. retry in 1.238923272s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:33.802805922Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2\") failed: Property not found. retry in 468.603195ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:33.814490904Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795\") failed: Property not found. retry in 745.802632ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:34.292105299Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2\") failed: Property not found. retry in 858.317213ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:34.299135536Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795\") failed: Property not found. retry in 774.953847ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:35.04684462Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:9fb2::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:35.049652621Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2303"}
{"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:35.049722772Z","hostname":"buskin","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"}
{"msg":"wait for service Some(\"oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490\") failed: Property not found. retry in 706.864869ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:35.055420655Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2\") failed: Property not found. retry in 713.253376ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:35.165690401Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe01:db42","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:35.60874822Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2310"}
{"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:35.615251628Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2371"}
{"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:35.740420313Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2556"}
{"msg":"wait for service Some(\"oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490\") failed: Property not found. retry in 830.303914ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:35.772177068Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795\") failed: Property not found. retry in 611.299086ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:35.831136185Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2\") failed: Property not found. retry in 1.025449849s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:35.890942118Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:36.000977508Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2720"}
{"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:36.200091472Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2435"}
{"msg":"wait for service Some(\"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795\") failed: Property not found. retry in 603.302096ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:36.45296947Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:36.523269862Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2470"}
{"msg":"wait for service Some(\"oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490\") failed: Property not found. retry in 1.319771206s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:36.615234257Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:36.743661631Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2758"}
{"msg":"wait for service Some(\"oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2\") failed: Property not found. retry in 1.271727719s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:36.925842599Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:36.931577043Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2785"}
{"msg":"wait for service Some(\"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795\") failed: Property not found. retry in 848.769965ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:37.065519047Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:37.142810035Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2664"}
{"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:37.31155853Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2464"}
{"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:37.321098563Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3843","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"}
{"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:37.321136293Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3859"}
{"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:37.321148034Z","hostname":"buskin","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"}
{"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:37.867007314Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3867"}
{"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:37.873662074Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3923"}
{"msg":"wait for service Some(\"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795\") failed: Property not found. retry in 1.088799286s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:37.922245994Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490\") failed: Property not found. retry in 1.446633536s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:37.942790705Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:38.005261066Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3985"}
{"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:38.101860724Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3966","rack_subnet":"fd00:1122:3344::"}
{"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:38.12262342Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:4008"}
{"msg":"wait for service Some(\"oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2\") failed: Property not found. retry in 1.301122391s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:38.207525767Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:38.217690871Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:4035"}
{"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:38.342583971Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:364"}
{"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:38.374724153Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:412"}
{"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:38.406412207Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:589"}
{"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:38.406491048Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:627"}
{"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:38.406903486Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:630","uptime":"3"}
{"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:38.406916116Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:434","config":"PortSettings {\n    links: {\n        \"0\": LinkSettings {\n            addrs: [\n                10.151.1.174,\n            ],\n            params: LinkCreate {\n                autoneg: false,\n                fec: None,\n                kr: false,\n                lane: Some(\n                    LinkId(\n                        0,\n                    ),\n                ),\n                speed: Speed40G,\n            },\n        },\n    },\n}"}
{"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:38.483023853Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:38.483102735Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:38.483114185Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:38.483121915Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:38.483131346Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:38.483137595Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:38.483146316Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:38.483153146Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:38.483158946Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:38.483171246Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:38.483176556Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:38.483181686Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"wait for service Some(\"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795\") failed: Property not found. retry in 740.087051ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:39.020160915Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490\") failed: Property not found. retry in 880.315473ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:39.399108354Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2\") failed: Property not found. retry in 1.073940868s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:39.51825697Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:40.610190284Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_internal_dns_55e16fed-0b39-4c9e-94aa-dbf1dfed9795"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:40.610254806Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_internal_dns_c037c9e2-43a2-432a-bcd7-352e08f26490"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:40.610261346Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_internal_dns_655db966-87f1-48aa-8868-000f036af1b2"}
{"msg":"Writing ledger to /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:40.610271756Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:40.610464459Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:40.610710174Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"8a190884-8f3c-4426-9051-8e939640cc87","remote_addr":"[fd00:1122:3344:101::1]:46225","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":11537666,"response_code":"204"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:40.612389854Z","hostname":"buskin","pid":1618,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:545"}
{"msg":"failed to write DNS configuration (will retry in 52.200636ms)","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:40.856196487Z","hostname":"buskin","pid":1618,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:565","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"}
{"msg":"failed to write DNS configuration (will retry in 83.757735ms)","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:40.910383658Z","hostname":"buskin","pid":1618,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:565","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.046058593Z","hostname":"buskin","pid":1618,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:545"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.124497593Z","hostname":"buskin","pid":1618,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:545"}
{"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.206281694Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:581"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.206295154Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:258"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.237158782Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.237888495Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:300"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.237924926Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.266198238Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.26633208Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:53993"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.266514704Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.266524774Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.286816861Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.286872482Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.306370385Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.306424866Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.324535384Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.324598605Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.324799649Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.32490511Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.336067592Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.367535112Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.368192734Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.368204434Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.411312534Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 143, 167]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.411349215Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:1277","zone_type":"ntp"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.1.169,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.411968536Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.1.169,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.411979857Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.1.169,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.412289132Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.41435737Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.439482154Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.459848943Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.819993572Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36446"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:41.820210846Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"aa7f435c-6b86-4065-9126-90715890719d","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36446","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":77,"response_code":"200"}
{"msg":"Profile for oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService4\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::1b\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"1.1.1.1\" />\n            <value_node value=\"9.9.9.9\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/ntp\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"allow\" value=\"fd00:1122:3344:100::/56\"/>\n        <propval type=\"boolean\" name=\"boundary\" value=\"true\"/>\n        <propval type=\"astring\" name=\"server\" value=\"0.pool.ntp.org\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte0\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.3.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.3.5\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:42.631895696Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:42.632006268Z","hostname":"buskin","pid":1618,"zone":"oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\") failed: Property not found. retry in 31.653846ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:42.758121691Z","hostname":"buskin","pid":1618,"zone":"oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\") failed: Property not found. retry in 144.713482ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:42.79783853Z","hostname":"buskin","pid":1618,"zone":"oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\") failed: Property not found. retry in 251.324635ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:42.967149914Z","hostname":"buskin","pid":1618,"zone":"oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\") failed: Property not found. retry in 457.55305ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:43.230071413Z","hostname":"buskin","pid":1618,"zone":"oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\") failed: Property not found. retry in 979.839814ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:43.698308728Z","hostname":"buskin","pid":1618,"zone":"oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\") failed: Property not found. retry in 714.503579ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:44.688069613Z","hostname":"buskin","pid":1618,"zone":"oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\") failed: Property not found. retry in 903.89542ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:45.411933564Z","hostname":"buskin","pid":1618,"zone":"oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\") failed: Property not found. retry in 1.06299245s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:46.324254137Z","hostname":"buskin","pid":1618,"zone":"oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\") failed: Property not found. retry in 516.508395ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:47.397088195Z","hostname":"buskin","pid":1618,"zone":"oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\") failed: Property not found. retry in 1.343568085s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:47.923936331Z","hostname":"buskin","pid":1618,"zone":"oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:49.286159596Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7"}
{"msg":"Writing ledger to /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:49.286273688Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:49.286558124Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:49.286843439Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"4cc65d6a-5c73-4dc3-ae4a-5ba7cea49cf5","remote_addr":"[fd00:1122:3344:101::1]:53993","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8020471,"response_code":"204"}
{"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:49.288623311Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:622"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:49.317163158Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:49.317383771Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:51691"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:49.325894526Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3574"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:49.326023518Z","hostname":"buskin","pid":1618,"uri":"/timesync","method":"GET","req_id":"aaeaf4a8-337e-41c0-9f5b-35cfc3c37d41","remote_addr":"[fd00:1122:3344:101::1]:51691","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8587,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:49.327445094Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:49.48633815Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:49.486432831Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:65502"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:53.494038988Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3574"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:53.49415445Z","hostname":"buskin","pid":1618,"uri":"/timesync","method":"GET","req_id":"4244cfa3-803d-4bfd-b5e3-cd5032f8aa55","remote_addr":"[fd00:1122:3344:101::1]:65502","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4007593,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:53.497055643Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:53.649682815Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:53.649810758Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:47720"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:53.658868422Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3574"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:53.659066035Z","hostname":"buskin","pid":1618,"uri":"/timesync","method":"GET","req_id":"819e485c-1545-4866-a6b0-9ea264430274","remote_addr":"[fd00:1122:3344:101::1]:47720","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9082,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:53.662046489Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:53.848686077Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:53.848773459Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:60365"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_2168e6f2-fbb0-4d6d-8811-9fd75847a2a7': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:57.857260671Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3574"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:57.857349103Z","hostname":"buskin","pid":1618,"uri":"/timesync","method":"GET","req_id":"58428d31-12a9-4d0e-b7ab-8d5b77336fef","remote_addr":"[fd00:1122:3344:101::1]:60365","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008451,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:57.860692593Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:57.997178884Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:57.997302276Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:39270"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:58.005579856Z","hostname":"buskin","pid":1618,"uri":"/timesync","method":"GET","req_id":"9838962e-fe87-442a-adfd-42f257f334b9","remote_addr":"[fd00:1122:3344:101::1]:39270","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8137,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1711777373.8469317, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:58.007652903Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:630"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:58.007663273Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:58.246514627Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:58.246599728Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:53043"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:58.255023921Z","hostname":"buskin","pid":1618,"uri":"/timesync","method":"GET","req_id":"97f80927-f5ee-4e21-a2c5-6c2eb2701f57","remote_addr":"[fd00:1122:3344:101::1]:53043","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8320,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1711777373.8469317, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:58.258373511Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:630"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:58.258378621Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:58.542951732Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:58.543052754Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:54815"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:58.551861413Z","hostname":"buskin","pid":1618,"uri":"/timesync","method":"GET","req_id":"cbe6f74d-e482-40c9-8e45-cf60ce71b022","remote_addr":"[fd00:1122:3344:101::1]:54815","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8720,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1711777373.8469317, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:58.55387992Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:630"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:58.55388701Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:58.810061957Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:58.810148488Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:37167"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:58.818223115Z","hostname":"buskin","pid":1618,"uri":"/timesync","method":"GET","req_id":"4b4dfd81-5131-4ef1-832e-46879cd169b2","remote_addr":"[fd00:1122:3344:101::1]:37167","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7962,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1711777373.8469317, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:58.820094048Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:630"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:58.820099219Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:59.289749398Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:59.289873121Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:38379"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:59.298045729Z","hostname":"buskin","pid":1618,"uri":"/timesync","method":"GET","req_id":"051f234a-16a3-4518-800b-c58ac496b72c","remote_addr":"[fd00:1122:3344:101::1]:38379","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8051,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1711777373.8469317, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:59.300132336Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:630"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:59.300139996Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:59.673285901Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:59.673392892Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:51457"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:59.681469718Z","hostname":"buskin","pid":1618,"uri":"/timesync","method":"GET","req_id":"be436a97-ca58-4b2e-b542-5ba71535f39e","remote_addr":"[fd00:1122:3344:101::1]:51457","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7989,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1711777373.8469317, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:42:59.684562705Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:630"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:42:59.684568215Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:00.042624355Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:00.042716857Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:65458"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:00.050845304Z","hostname":"buskin","pid":1618,"uri":"/timesync","method":"GET","req_id":"ca934cb7-099b-4dc0-9b53-b8aa8fb29ad4","remote_addr":"[fd00:1122:3344:101::1]:65458","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8057,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1711777373.8469317, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:00.054317337Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:630"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:00.054322497Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.476055026Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.476141778Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:52940"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.479079751Z","hostname":"buskin","pid":1618,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.495795484Z","hostname":"buskin","pid":1618,"uri":"/timesync","method":"GET","req_id":"9a1c80ea-9022-455c-9944-f0288f7180e8","remote_addr":"[fd00:1122:3344:101::1]:52940","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":19503,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 843850428, ip_addr: 50.76.34.188, stratum: 2, ref_time: 1711777380.5411906, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.497735169Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:630"}
{"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.497740849Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1210"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.526127983Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.526301066Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:48593"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.52651365Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.52652454Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.54531777Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.545353041Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.56518198Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.56523086Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.583595043Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.583639354Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.660551206Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.660599187Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.734347932Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.734393582Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.812762701Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.812808872Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.884824875Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.884897446Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.958133442Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.958191143Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.958407107Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.958530089Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:01.991426625Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:01.991484706Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:01.991494626Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:01.991503626Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:01.991512196Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:01.991658439Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"eeaa1cae-8957-4cf0-976c-65edc5fdfbe7","remote_addr":"[fd00:1122:3344:101::1]:48593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465\", TimeNotSynchronized), (\"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349\", TimeNotSynchronized), (\"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198\", TimeNotSynchronized), (\"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306\", TimeNotSynchronized), (\"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f\", TimeNotSynchronized)]","latency_us":465280,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:01.991917783Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"98.048812ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"eeaa1cae-8957-4cf0-976c-65edc5fdfbe7\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:43:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"eeaa1cae-8957-4cf0-976c-65edc5fdfbe7\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.091074048Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.091650759Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.091676869Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.110752085Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.110799725Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.128274111Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.128322443Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.145990032Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.146039783Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.165314192Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.165356233Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.184025811Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.184064892Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.202591377Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.202652658Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.220012632Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.220055643Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.237255584Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.237308415Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.237518939Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.237669962Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:02.270266712Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:02.270313382Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:02.270330263Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:02.270337323Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:02.270345843Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.270451715Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"783397ea-524e-4639-ad29-34566a34b49f","remote_addr":"[fd00:1122:3344:101::1]:48593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198\", TimeNotSynchronized), (\"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306\", TimeNotSynchronized), (\"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465\", TimeNotSynchronized), (\"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f\", TimeNotSynchronized), (\"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349\", TimeNotSynchronized)]","latency_us":179033,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:02.270630358Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"137.241429ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"783397ea-524e-4639-ad29-34566a34b49f\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:43:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"783397ea-524e-4639-ad29-34566a34b49f\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.40887445Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.409332259Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.409348029Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.427172192Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.427224003Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.444106928Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.444157799Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.46299421Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.463037551Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.48123503Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.481280911Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.498400351Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.498439652Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.51550524Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.515547461Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.532565319Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.53260348Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.007475576Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.007535646Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.007686689Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.007807611Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.010013691Z","hostname":"buskin","pid":1618,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.089782655Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.089857887Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.089868777Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.089877387Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.089884457Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.178468471Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.182758088Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.187058046Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.191519457Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.19612083Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.200649122Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.204911129Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.209093715Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.213591946Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:02.218130098Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService7\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::4\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::4\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:05.963047859Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:05.963312744Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:05.966614244Z","hostname":"buskin","pid":1618,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"}
{"msg":"Profile for oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService9\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::3\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::3\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:05.969478715Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:05.969620248Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349\") failed: Property not found. retry in 74.98675ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:06.11613287Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306\") failed: Property not found. retry in 45.151757ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:06.123921171Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService6\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::6\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::6\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:06.161001712Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:06.161159375Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306\") failed: Property not found. retry in 83.345683ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:06.18301056Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349\") failed: Property not found. retry in 87.058181ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:06.279284123Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306\") failed: Property not found. retry in 201.314312ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:06.35423403Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService8\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::5\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::5\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:06.364805521Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:06.365450453Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService5\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::7\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::7\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:06.386805679Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349\") failed: Property not found. retry in 189.714903ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:06.519775936Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:06.519946119Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306\") failed: Property not found. retry in 470.906115ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:06.64763047Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465\") failed: Property not found. retry in 40.57049ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:06.676490202Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349\") failed: Property not found. retry in 261.241756ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:06.694996737Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465\") failed: Property not found. retry in 121.406801ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:06.849067916Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349\") failed: Property not found. retry in 798.772729ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:07.038688038Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465\") failed: Property not found. retry in 113.975802ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:07.066273207Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198\") failed: Property not found. retry in 41.497543ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:07.121265113Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306\") failed: Property not found. retry in 690.772957ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:07.145476531Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f\") failed: Property not found. retry in 60.376009ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:07.167159564Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198\") failed: Property not found. retry in 50.889724ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:07.190392584Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465\") failed: Property not found. retry in 418.014792ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:07.517721589Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198\") failed: Property not found. retry in 133.974657ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:07.678307055Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f\") failed: Property not found. retry in 83.771439ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:07.749765049Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198\") failed: Property not found. retry in 514.750444ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:07.838077317Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306\") failed: Property not found. retry in 1.295862068s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:07.871285428Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f\") failed: Property not found. retry in 131.981885ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:07.927905343Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349\") failed: Property not found. retry in 662.602625ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:07.996373972Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465\") failed: Property not found. retry in 1.029241152s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:08.095063459Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f\") failed: Property not found. retry in 242.307082ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:08.253616738Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198\") failed: Property not found. retry in 691.111938ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:08.496392283Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f\") failed: Property not found. retry in 1.014682526s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:08.530217155Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349\") failed: Property not found. retry in 1.156044005s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:08.671007003Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465\") failed: Property not found. retry in 1.252827806s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:09.136205293Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306\") failed: Property not found. retry in 1.374148265s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:09.17800858Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198\") failed: Property not found. retry in 843.35988ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:09.199774234Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f\") failed: Property not found. retry in 657.966386ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:09.555674756Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349\") failed: Property not found. retry in 1.098610591s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:09.836778874Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198\") failed: Property not found. retry in 685.393483ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:10.052113921Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f\") failed: Property not found. retry in 729.104776ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:10.22388367Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465\") failed: Property not found. retry in 540.587494ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:10.397961081Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306\") failed: Property not found. retry in 915.432079ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:10.561387929Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198\") failed: Property not found. retry in 836.161216ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:10.747245993Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349\") failed: Property not found. retry in 1.185840432s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:10.94876384Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465\") failed: Property not found. retry in 1.475314142s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:10.960179587Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f\") failed: Property not found. retry in 1.171772371s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:10.970926661Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306\") failed: Property not found. retry in 648.46544ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:11.488609942Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198\") failed: Property not found. retry in 837.432914ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:11.595434245Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349\") failed: Property not found. retry in 622.382507ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:12.144110135Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306\") failed: Property not found. retry in 1.159309292s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:12.152177831Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f\") failed: Property not found. retry in 1.017883099s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:12.159577305Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198\") failed: Property not found. retry in 961.919317ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:12.441413767Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465\") failed: Property not found. retry in 826.202601ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:12.448645387Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:12.543213639Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64015"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:12.543351062Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"e55e0489-0783-41c6-82af-6991f8d16b8c","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64015","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":76,"response_code":"200"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349\") failed: Property not found. retry in 662.084947ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:12.775242409Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f\") failed: Property not found. retry in 1.022508166s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:13.186253538Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465\") failed: Property not found. retry in 651.502242ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:13.283779313Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306\") failed: Property not found. retry in 1.498419787s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:13.320096171Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198\") failed: Property not found. retry in 566.447217ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:13.411143568Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349\") failed: Property not found. retry in 1.195305009s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:13.445403049Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465\") failed: Property not found. retry in 509.492856ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:13.943767909Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198\") failed: Property not found. retry in 1.167490939s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:13.986437681Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f\") failed: Property not found. retry in 507.968133ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:14.217651726Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465\") failed: Property not found. retry in 1.242041898s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:14.46207068Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349\") failed: Property not found. retry in 742.452379ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:14.654257679Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f\") failed: Property not found. retry in 505.294437ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:14.736711561Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306\") failed: Property not found. retry in 986.356327ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:14.829893738Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f\") failed: Property not found. retry in 535.061675ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:15.251518639Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465\") failed: Property not found. retry in 654.650963ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:15.714259874Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f\") failed: Property not found. retry in 865.161808ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:15.797113254Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306\") failed: Property not found. retry in 1.149466414s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:15.825916835Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465\") failed: Property not found. retry in 826.346364ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:16.37976572Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f\") failed: Property not found. retry in 943.234757ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:16.673746541Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306\") failed: Property not found. retry in 1.128632566s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:17.101688646Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465\") failed: Property not found. retry in 1.228582519s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:17.224604121Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f\") failed: Property not found. retry in 1.184607809s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:17.629380398Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306\") failed: Property not found. retry in 1.397374848s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:18.243155027Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465\") failed: Property not found. retry in 1.309149493s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:18.464718127Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f\") failed: Property not found. retry in 1.157812974s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:18.827447142Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306\") failed: Property not found. retry in 775.046184ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:19.650330497Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465\") failed: Property not found. retry in 690.06206ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:19.782342336Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f\") failed: Property not found. retry in 1.402281868s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:19.99358266Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306\") failed: Property not found. retry in 568.225479ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:20.433850949Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465\") failed: Property not found. retry in 779.229634ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:20.482354916Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306\") failed: Property not found. retry in 1.377831532s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:21.01126885Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465\") failed: Property not found. retry in 1.273797986s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:21.270349089Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f\") failed: Property not found. retry in 837.841395ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:21.403963188Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f\") failed: Property not found. retry in 594.671559ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:22.251835454Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306\") failed: Property not found. retry in 594.915466ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:22.398505809Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465\") failed: Property not found. retry in 852.9651ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:22.55216363Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f\") failed: Property not found. retry in 1.099097027s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:22.8555128Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306\") failed: Property not found. retry in 987.15692ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:23.002363458Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465\") failed: Property not found. retry in 905.94825ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:23.413277915Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f\") failed: Property not found. retry in 1.457672751s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:23.96328595Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306\") failed: Property not found. retry in 813.025857ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:23.999022147Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465\") failed: Property not found. retry in 634.542485ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:24.327680856Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306\") failed: Property not found. retry in 1.328548039s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:24.821417272Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:26.171454638Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_cockroachdb_f7311d36-617f-4f72-be7a-3427dc69c306"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:26.171535529Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_cockroachdb_f7108ea5-965b-4e70-a9a0-7ffd5d13885f"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:26.17154363Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:26.17155071Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_cockroachdb_f597bf55-7c6a-456c-8e41-286fd5037349"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:26.1715555Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_cockroachdb_ddf7c878-622c-4bc8-8b0b-5a158ff3b198"}
{"msg":"Writing ledger to /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:26.17156772Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:26.172012638Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:26.172334224Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"24bcb46e-feca-4962-9b5a-57786ee3c9cb","remote_addr":"[fd00:1122:3344:101::1]:48593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":24012181,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:26.206623885Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:52474"}
{"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::6]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:26.236375283Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3408"}
{"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:35.453299397Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3426"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:36.976772511Z","hostname":"buskin","pid":1618,"uri":"/cockroachdb","method":"POST","req_id":"23af3898-6dae-4aea-977f-32b6ec174f42","remote_addr":"[fd00:1122:3344:101::1]:52474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_9d0139eb-7e07-43fa-92d7-f8a6676bf465': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::6]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1  stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE VIEW\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE VIEW\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE VIEW\nCREATE VIEW\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\n  stderr: ERROR: restart transaction: TransactionRetryWithProtoRefreshError: TransactionAbortedError(ABORT_REASON_ABORT_SPAN): \"sql txn\" meta={id=dbb6cfe3 key=/NamespaceTable/30/1/106/0/\"public\"/4/1 pri=0.08421992 epo=0 ts=1711777416.875419487,1 min=1711777416.375517229,0 seq=702} lock=true stat=ABORTED rts=1711777416.375517229,0 wto=false gul=1711777416.875517229,0\nSQLSTATE: 40001\nHINT: See: https://www.cockroachlabs.com/docs/v22.1/transaction-retry-error-reference.html#abort_reason_abort_span\nERROR: restart transaction: TransactionRetryWithProtoRefreshError: TransactionAbortedError(ABORT_REASON_ABORT_SPAN): \"sql txn\" meta={id=dbb6cfe3 key=/NamespaceTable/30/1/106/0/\"public\"/4/1 pri=0.08421992 epo=0 ts=1711777416.875419487,1 min=1711777416.375517229,0 seq=702} lock=true stat=ABORTED rts=1711777416.375517229,0 wto=false gul=1711777416.875517229,0\nSQLSTATE: 40001\nHINT: See: https://www.cockroachlabs.com/docs/v22.1/transaction-retry-error-reference.html#abort_reason_abort_span\nFailed running \"sql\"\n","latency_us":10769600,"response_code":"500"}
{"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:36.984313628Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:952","retry_after":"77.403016ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"23af3898-6dae-4aea-977f-32b6ec174f42\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:43:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"23af3898-6dae-4aea-977f-32b6ec174f42\" }"}
{"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::6]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:37.128562719Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3408"}
{"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:37.286429816Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3426"}
{"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:40.875907965Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3449"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:40.876016116Z","hostname":"buskin","pid":1618,"uri":"/cockroachdb","method":"POST","req_id":"5cde89ed-9b00-46a6-b7fb-ebc7181a4d96","remote_addr":"[fd00:1122:3344:101::1]:52474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3811892,"response_code":"204"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 57425cdc-844b-42e3-b45f-c8fb7871161f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b6a17b0f-0906-4eb9-94c3-528d000f68dc, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f4d5df0a-748d-40ab-8235-7c927aae50ce, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 23c9f9de-3736-4878-ba3b-118cebaacb82, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a684a100-6031-4b97-8de3-41e15037fbd5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 648e5f66-03f8-489e-a2bb-91cee8654a72, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 80d3c328-30b9-4b19-852b-d0002f908859, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 06241231-d15c-4adb-ae5b-e8e364e35528, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } } } }, OmicronZoneConfig { id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } } } }, OmicronZoneConfig { id: 57f3dc22-0a8a-40ec-a7b7-137531501134, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } } } }, OmicronZoneConfig { id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:40.940540095Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:40.941850158Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:52853"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:40.944477856Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:40.944487166Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:40.970255102Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:40.970319294Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:40.989969459Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:40.99002428Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.009521383Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.009569164Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.029348312Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.029389153Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.04856751Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.048608751Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.06955701Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.069611491Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.088840309Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.088882529Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.108027626Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.108072867Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.182714128Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.182771069Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.262236427Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.262296358Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.337008761Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.337064602Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.416495539Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.416560751Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.48725715Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.487307431Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.565364514Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.565413235Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.651836989Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.6518971Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.728369704Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.728438656Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.79718103Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.797235631Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.857522742Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.857585013Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.921565211Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.921619162Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.98449226Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.984796496Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.985375566Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:41.98556257Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.104885719Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.10494962Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.10495682Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.10496316Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.104970731Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.104979031Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.104987821Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.104994461Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.105003881Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.105011801Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.105019302Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.105025532Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.105032502Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.105048122Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.105055202Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.105060922Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.105066362Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.105072082Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.105078993Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.105191275Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"2d4b2754-f3de-45ba-b2f2-9dbfe727611b","remote_addr":"[fd00:1122:3344:101::1]:52853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\", TimeNotSynchronized), (\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\", TimeNotSynchronized), (\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\", TimeNotSynchronized), (\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\", TimeNotSynchronized), (\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\", TimeNotSynchronized), (\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\", TimeNotSynchronized), (\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\", TimeNotSynchronized), (\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\", TimeNotSynchronized), (\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\", TimeNotSynchronized), (\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\", TimeNotSynchronized), (\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\", TimeNotSynchronized), (\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\", TimeNotSynchronized), (\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\", TimeNotSynchronized), (\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\", TimeNotSynchronized), (\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\", TimeNotSynchronized), (\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\", TimeNotSynchronized), (\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\", TimeNotSynchronized), (\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\", TimeNotSynchronized), (\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\", TimeNotSynchronized)]","latency_us":1163279,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.105555541Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"134.065463ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2d4b2754-f3de-45ba-b2f2-9dbfe727611b\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:43:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2d4b2754-f3de-45ba-b2f2-9dbfe727611b\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 57425cdc-844b-42e3-b45f-c8fb7871161f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b6a17b0f-0906-4eb9-94c3-528d000f68dc, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f4d5df0a-748d-40ab-8235-7c927aae50ce, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 23c9f9de-3736-4878-ba3b-118cebaacb82, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a684a100-6031-4b97-8de3-41e15037fbd5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 648e5f66-03f8-489e-a2bb-91cee8654a72, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 80d3c328-30b9-4b19-852b-d0002f908859, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 06241231-d15c-4adb-ae5b-e8e364e35528, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } } } }, OmicronZoneConfig { id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } } } }, OmicronZoneConfig { id: 57f3dc22-0a8a-40ec-a7b7-137531501134, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } } } }, OmicronZoneConfig { id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.241142065Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.242425439Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.242437009Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.263952558Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.263997549Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.285255964Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.285305615Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.305147064Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.305191655Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.325526543Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.325571374Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.345432703Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.345479844Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.366047176Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.366104047Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.386048778Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.386097599Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.405268396Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.405316497Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.424538515Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.424587366Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.445091007Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.445138198Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.465770611Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.465827882Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.486520507Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.486563498Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.50657747Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.506624421Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.52593808Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.525985831Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.544341573Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.544479886Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.564915166Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.564956526Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.584063122Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.584111603Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.588505673Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63548"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.58943874Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"9c900e18-88fc-4d89-9878-a2a66fee50be","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63548","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":179,"response_code":"200"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.604348179Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.60439646Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.622887145Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.622932146Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.642056032Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.642117583Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.642334477Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.642591852Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.73704008Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.737088941Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.737097421Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.737102172Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.737108252Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.737112512Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.737118482Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.737122842Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.737128322Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.737133842Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.737139512Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.737163013Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.737169253Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.737174643Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.737180253Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.737186133Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.737191833Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.737197613Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.737203613Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.737360386Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"299aaf71-e852-416f-a85e-303d9387bce9","remote_addr":"[fd00:1122:3344:101::1]:52853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\", TimeNotSynchronized), (\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\", TimeNotSynchronized), (\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\", TimeNotSynchronized), (\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\", TimeNotSynchronized), (\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\", TimeNotSynchronized), (\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\", TimeNotSynchronized), (\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\", TimeNotSynchronized), (\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\", TimeNotSynchronized), (\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\", TimeNotSynchronized), (\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\", TimeNotSynchronized), (\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\", TimeNotSynchronized), (\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\", TimeNotSynchronized), (\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\", TimeNotSynchronized), (\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\", TimeNotSynchronized), (\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\", TimeNotSynchronized), (\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\", TimeNotSynchronized), (\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\", TimeNotSynchronized), (\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\", TimeNotSynchronized), (\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\", TimeNotSynchronized)]","latency_us":495347,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:42.73757572Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"165.217808ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"299aaf71-e852-416f-a85e-303d9387bce9\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:43:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"299aaf71-e852-416f-a85e-303d9387bce9\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 57425cdc-844b-42e3-b45f-c8fb7871161f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b6a17b0f-0906-4eb9-94c3-528d000f68dc, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f4d5df0a-748d-40ab-8235-7c927aae50ce, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 23c9f9de-3736-4878-ba3b-118cebaacb82, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a684a100-6031-4b97-8de3-41e15037fbd5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 648e5f66-03f8-489e-a2bb-91cee8654a72, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 80d3c328-30b9-4b19-852b-d0002f908859, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 06241231-d15c-4adb-ae5b-e8e364e35528, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } } } }, OmicronZoneConfig { id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } } } }, OmicronZoneConfig { id: 57f3dc22-0a8a-40ec-a7b7-137531501134, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } } } }, OmicronZoneConfig { id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.904046043Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.90494521Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.90495523Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.925720466Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.925774547Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.944671529Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.944710109Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.960752469Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.96079207Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.980620439Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.98065755Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.997347782Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:42.997388033Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.015306477Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.015345488Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.033424205Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.033472066Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.050532765Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.050575085Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.06849197Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.06852686Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.085354745Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.085396866Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.102531306Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.102573507Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.119438592Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.119483533Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.136380248Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.136413829Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.153761983Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.153800174Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.170850272Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.170895223Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.187615076Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.187655987Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.204918529Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.2049588Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.221681262Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.221718563Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.241071493Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.241130544Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.25858633Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.258654722Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.258874305Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.259022768Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:43.346290168Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:43.346340279Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:43.346348389Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:43.346355689Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:43.346364279Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:43.346373329Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:43.346380369Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:43.346387249Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:43.34639225Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:43.34639897Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:43.34640749Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:43.34641404Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:43.34643014Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:43.346438881Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:43.346447521Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:43.346455811Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:43.346462631Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:43.346471151Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:43.346477691Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.346605884Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"18086550-6dfa-40bc-9369-7900ce051b76","remote_addr":"[fd00:1122:3344:101::1]:52853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\", TimeNotSynchronized), (\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\", TimeNotSynchronized), (\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\", TimeNotSynchronized), (\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\", TimeNotSynchronized), (\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\", TimeNotSynchronized), (\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\", TimeNotSynchronized), (\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\", TimeNotSynchronized), (\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\", TimeNotSynchronized), (\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\", TimeNotSynchronized), (\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\", TimeNotSynchronized), (\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\", TimeNotSynchronized), (\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\", TimeNotSynchronized), (\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\", TimeNotSynchronized), (\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\", TimeNotSynchronized), (\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\", TimeNotSynchronized), (\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\", TimeNotSynchronized), (\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\", TimeNotSynchronized), (\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\", TimeNotSynchronized), (\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\", TimeNotSynchronized)]","latency_us":442013,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:43.346851628Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"201.729238ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"18086550-6dfa-40bc-9369-7900ce051b76\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:43:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"18086550-6dfa-40bc-9369-7900ce051b76\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 57425cdc-844b-42e3-b45f-c8fb7871161f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b6a17b0f-0906-4eb9-94c3-528d000f68dc, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f4d5df0a-748d-40ab-8235-7c927aae50ce, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 23c9f9de-3736-4878-ba3b-118cebaacb82, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a684a100-6031-4b97-8de3-41e15037fbd5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 648e5f66-03f8-489e-a2bb-91cee8654a72, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 80d3c328-30b9-4b19-852b-d0002f908859, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 06241231-d15c-4adb-ae5b-e8e364e35528, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } } } }, OmicronZoneConfig { id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } } } }, OmicronZoneConfig { id: 57f3dc22-0a8a-40ec-a7b7-137531501134, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } } } }, OmicronZoneConfig { id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.55025529Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.550754189Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.550761829Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.571228829Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.57128664Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.589607772Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.589657773Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.608399822Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.608450683Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.630689615Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.630738846Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.650919202Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.650968842Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.670805011Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.670863182Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.688141375Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.688191176Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.706246853Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.706283863Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.724259839Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.724302889Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.742242934Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.742287575Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.759912274Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.759957025Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.777074585Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.777120726Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.794439109Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.79448382Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.813119737Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.813160468Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.830539132Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.830576353Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.848542158Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.848588859Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.866059605Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.866101306Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.884330246Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.884373167Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.901605619Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.90165056Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.920261307Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.920314328Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.920539121Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:43.920682344Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.007854382Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.007904153Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.007912683Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.007920163Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.007926093Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.007932303Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.007938054Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.007944054Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.007949974Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.007955774Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.007962864Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.007968554Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.007975484Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.007997484Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.008003615Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.008009055Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.008014505Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.008019975Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.008026725Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.008151747Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"dca4100b-b0bf-433b-8b37-d40458fd862c","remote_addr":"[fd00:1122:3344:101::1]:52853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\", TimeNotSynchronized), (\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\", TimeNotSynchronized), (\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\", TimeNotSynchronized), (\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\", TimeNotSynchronized), (\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\", TimeNotSynchronized), (\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\", TimeNotSynchronized), (\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\", TimeNotSynchronized), (\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\", TimeNotSynchronized), (\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\", TimeNotSynchronized), (\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\", TimeNotSynchronized), (\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\", TimeNotSynchronized), (\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\", TimeNotSynchronized), (\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\", TimeNotSynchronized), (\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\", TimeNotSynchronized), (\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\", TimeNotSynchronized), (\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\", TimeNotSynchronized), (\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\", TimeNotSynchronized), (\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\", TimeNotSynchronized), (\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\", TimeNotSynchronized)]","latency_us":457578,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.008975562Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"242.040673ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dca4100b-b0bf-433b-8b37-d40458fd862c\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:43:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dca4100b-b0bf-433b-8b37-d40458fd862c\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 57425cdc-844b-42e3-b45f-c8fb7871161f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b6a17b0f-0906-4eb9-94c3-528d000f68dc, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f4d5df0a-748d-40ab-8235-7c927aae50ce, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 23c9f9de-3736-4878-ba3b-118cebaacb82, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a684a100-6031-4b97-8de3-41e15037fbd5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 648e5f66-03f8-489e-a2bb-91cee8654a72, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 80d3c328-30b9-4b19-852b-d0002f908859, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 06241231-d15c-4adb-ae5b-e8e364e35528, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } } } }, OmicronZoneConfig { id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } } } }, OmicronZoneConfig { id: 57f3dc22-0a8a-40ec-a7b7-137531501134, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } } } }, OmicronZoneConfig { id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.252347347Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.252810706Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.252817586Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.274249364Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.274294235Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.293109235Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.293149986Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.312774121Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.312817082Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.331098263Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.331139763Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.34972619Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.34976715Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.367276727Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.367330868Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.384342596Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.384386307Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.401512027Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.401565328Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.419614715Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.419655855Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.437279054Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.437326225Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.455385242Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.455441973Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.4729286Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.472981681Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.491669509Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.49171532Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.510354667Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.510400098Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.530209456Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.530254317Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.550112367Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.550161988Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.569015369Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.56905931Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.589908607Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.589959098Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.609280018Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.609322028Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.629129367Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.629182838Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.629384552Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.629512494Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.718900982Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.718950083Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.718959743Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.718966443Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.718975253Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.718981883Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.718988583Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.718996333Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.719003024Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.719010524Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.719017314Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.719023894Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.719030564Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.719037294Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.719042184Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.719059945Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.719068575Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.719074705Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.719082915Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.719228318Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"9e90e225-17ed-4109-aeb9-4c6371ab595a","remote_addr":"[fd00:1122:3344:101::1]:52853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\", TimeNotSynchronized), (\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\", TimeNotSynchronized), (\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\", TimeNotSynchronized), (\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\", TimeNotSynchronized), (\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\", TimeNotSynchronized), (\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\", TimeNotSynchronized), (\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\", TimeNotSynchronized), (\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\", TimeNotSynchronized), (\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\", TimeNotSynchronized), (\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\", TimeNotSynchronized), (\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\", TimeNotSynchronized), (\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\", TimeNotSynchronized), (\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\", TimeNotSynchronized), (\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\", TimeNotSynchronized), (\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\", TimeNotSynchronized), (\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\", TimeNotSynchronized), (\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\", TimeNotSynchronized), (\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\", TimeNotSynchronized), (\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\", TimeNotSynchronized)]","latency_us":466587,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:44.719468932Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"177.26639ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9e90e225-17ed-4109-aeb9-4c6371ab595a\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:43:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9e90e225-17ed-4109-aeb9-4c6371ab595a\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 57425cdc-844b-42e3-b45f-c8fb7871161f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b6a17b0f-0906-4eb9-94c3-528d000f68dc, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f4d5df0a-748d-40ab-8235-7c927aae50ce, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 23c9f9de-3736-4878-ba3b-118cebaacb82, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a684a100-6031-4b97-8de3-41e15037fbd5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 648e5f66-03f8-489e-a2bb-91cee8654a72, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 80d3c328-30b9-4b19-852b-d0002f908859, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 06241231-d15c-4adb-ae5b-e8e364e35528, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } } } }, OmicronZoneConfig { id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } } } }, OmicronZoneConfig { id: 57f3dc22-0a8a-40ec-a7b7-137531501134, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } } } }, OmicronZoneConfig { id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.897879111Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.899258166Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.899264507Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.918574566Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.918611507Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.93704584Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.937093561Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.955007405Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.955070697Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.972664805Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.972707576Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.989989098Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:44.990036129Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.007000737Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.007048537Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.026010521Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.026050721Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.044395453Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.044433784Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.062490511Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.062526982Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.080740971Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.080781142Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.09835661Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.098396791Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.115721314Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.115900188Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.133030958Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.133072328Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.150670047Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.150716678Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.168422908Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.168465479Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.186441674Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.186487675Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.205440678Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.205482959Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.222756442Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.222814113Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.240003514Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.240041995Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.257567392Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.257608593Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.257856227Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.25800036Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:45.353752753Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:45.353814464Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:45.353821804Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:45.353829824Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:45.353837754Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:45.353844144Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:45.353852255Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:45.353858205Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:45.353863145Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:45.353870975Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:45.353877045Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:45.353883375Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:45.353889255Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:45.353895265Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:45.353901315Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:45.353908516Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:45.353933226Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:45.353939736Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:45.353946736Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.354074388Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"74d41e00-0b2d-48ef-be6a-7bff5405f9c6","remote_addr":"[fd00:1122:3344:101::1]:52853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\", TimeNotSynchronized), (\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\", TimeNotSynchronized), (\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\", TimeNotSynchronized), (\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\", TimeNotSynchronized), (\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\", TimeNotSynchronized), (\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\", TimeNotSynchronized), (\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\", TimeNotSynchronized), (\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\", TimeNotSynchronized), (\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\", TimeNotSynchronized), (\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\", TimeNotSynchronized), (\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\", TimeNotSynchronized), (\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\", TimeNotSynchronized), (\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\", TimeNotSynchronized), (\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\", TimeNotSynchronized), (\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\", TimeNotSynchronized), (\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\", TimeNotSynchronized), (\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\", TimeNotSynchronized), (\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\", TimeNotSynchronized), (\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\", TimeNotSynchronized)]","latency_us":455713,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:45.354253372Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"156.421207ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"74d41e00-0b2d-48ef-be6a-7bff5405f9c6\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:43:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"74d41e00-0b2d-48ef-be6a-7bff5405f9c6\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 57425cdc-844b-42e3-b45f-c8fb7871161f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b6a17b0f-0906-4eb9-94c3-528d000f68dc, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f4d5df0a-748d-40ab-8235-7c927aae50ce, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 23c9f9de-3736-4878-ba3b-118cebaacb82, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a684a100-6031-4b97-8de3-41e15037fbd5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 648e5f66-03f8-489e-a2bb-91cee8654a72, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 80d3c328-30b9-4b19-852b-d0002f908859, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 06241231-d15c-4adb-ae5b-e8e364e35528, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } } } }, OmicronZoneConfig { id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } } } }, OmicronZoneConfig { id: 57f3dc22-0a8a-40ec-a7b7-137531501134, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } } } }, OmicronZoneConfig { id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.512473486Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.513399652Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.513406772Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.536009961Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.536050712Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.554776491Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.554831162Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.575912484Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.575956745Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.595369076Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.595411537Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.615638903Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.615681234Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.637385167Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.637434227Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.658250964Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.658296335Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.680422985Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.680466266Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.703154437Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.703193338Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.726822275Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.726870996Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.749242821Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.749294202Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.771512814Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.771561635Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.793559483Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.793606754Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.815163644Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.815210885Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.836683184Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.836728035Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.857839947Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.857898928Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.879267514Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.879312415Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.899872518Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.899915888Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.922077369Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.92212456Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.943488967Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.943554858Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.943717861Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:45.943836213Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.044265311Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.044320122Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.044345773Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.044350382Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.044355943Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.044361783Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.044367573Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.044371873Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.044379063Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.044384693Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.044392673Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.044398333Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.044405143Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.044409534Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.044416424Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.044421844Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.044427684Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.044440184Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.044445724Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.044539386Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"76d0150b-21e1-44b1-a704-4154d72dcb84","remote_addr":"[fd00:1122:3344:101::1]:52853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\", TimeNotSynchronized), (\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\", TimeNotSynchronized), (\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\", TimeNotSynchronized), (\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\", TimeNotSynchronized), (\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\", TimeNotSynchronized), (\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\", TimeNotSynchronized), (\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\", TimeNotSynchronized), (\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\", TimeNotSynchronized), (\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\", TimeNotSynchronized), (\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\", TimeNotSynchronized), (\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\", TimeNotSynchronized), (\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\", TimeNotSynchronized), (\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\", TimeNotSynchronized), (\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\", TimeNotSynchronized), (\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\", TimeNotSynchronized), (\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\", TimeNotSynchronized), (\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\", TimeNotSynchronized), (\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\", TimeNotSynchronized), (\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\", TimeNotSynchronized)]","latency_us":531802,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.04474387Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"384.158462ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"76d0150b-21e1-44b1-a704-4154d72dcb84\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:43:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"76d0150b-21e1-44b1-a704-4154d72dcb84\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 57425cdc-844b-42e3-b45f-c8fb7871161f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b6a17b0f-0906-4eb9-94c3-528d000f68dc, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f4d5df0a-748d-40ab-8235-7c927aae50ce, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 23c9f9de-3736-4878-ba3b-118cebaacb82, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a684a100-6031-4b97-8de3-41e15037fbd5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 648e5f66-03f8-489e-a2bb-91cee8654a72, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 80d3c328-30b9-4b19-852b-d0002f908859, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 06241231-d15c-4adb-ae5b-e8e364e35528, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } } } }, OmicronZoneConfig { id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } } } }, OmicronZoneConfig { id: 57f3dc22-0a8a-40ec-a7b7-137531501134, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } } } }, OmicronZoneConfig { id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.430230107Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.431301306Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.431314867Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.455448603Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.455502194Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.477462362Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.477514183Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.500853115Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.500902306Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.522619929Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.52267502Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.543148151Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.543193192Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.564922535Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.564971096Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.586263911Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.586309952Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.608405742Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.608456603Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.631382368Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.631436799Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.655242Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.655296641Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.677078505Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.677127396Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.697592896Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.697645307Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.717877963Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.717925794Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.739624537Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.739670838Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.761608685Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.761653476Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.782558334Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.782603365Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.802918092Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.802984094Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.825554542Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.825710155Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.845829229Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.84587895Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.866526784Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.866589575Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.866841589Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.867031973Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.967780907Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.967847988Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.967854918Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.967869018Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.967876568Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.967882148Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.967889098Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.967894659Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.967898779Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.967905689Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.967911119Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.967917029Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.967923819Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.967927929Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.967931739Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.967937249Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.967941079Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.96794491Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.96795891Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:46.968024101Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"bab56f20-9f85-4379-8450-24230bd5449e","remote_addr":"[fd00:1122:3344:101::1]:52853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\", TimeNotSynchronized), (\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\", TimeNotSynchronized), (\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\", TimeNotSynchronized), (\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\", TimeNotSynchronized), (\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\", TimeNotSynchronized), (\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\", TimeNotSynchronized), (\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\", TimeNotSynchronized), (\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\", TimeNotSynchronized), (\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\", TimeNotSynchronized), (\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\", TimeNotSynchronized), (\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\", TimeNotSynchronized), (\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\", TimeNotSynchronized), (\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\", TimeNotSynchronized), (\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\", TimeNotSynchronized), (\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\", TimeNotSynchronized), (\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\", TimeNotSynchronized), (\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\", TimeNotSynchronized), (\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\", TimeNotSynchronized), (\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\", TimeNotSynchronized)]","latency_us":537477,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:46.968239795Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"318.517218ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bab56f20-9f85-4379-8450-24230bd5449e\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:43:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bab56f20-9f85-4379-8450-24230bd5449e\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 57425cdc-844b-42e3-b45f-c8fb7871161f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b6a17b0f-0906-4eb9-94c3-528d000f68dc, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f4d5df0a-748d-40ab-8235-7c927aae50ce, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 23c9f9de-3736-4878-ba3b-118cebaacb82, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a684a100-6031-4b97-8de3-41e15037fbd5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 648e5f66-03f8-489e-a2bb-91cee8654a72, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 80d3c328-30b9-4b19-852b-d0002f908859, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 06241231-d15c-4adb-ae5b-e8e364e35528, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } } } }, OmicronZoneConfig { id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } } } }, OmicronZoneConfig { id: 57f3dc22-0a8a-40ec-a7b7-137531501134, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } } } }, OmicronZoneConfig { id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.287941151Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.288458381Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.288468871Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.312282892Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.312339473Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.332635271Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.332693612Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.352007561Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.352074982Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.372250877Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.372302118Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.391528437Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.391579847Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.4093956Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.409451941Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.426827355Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.426884456Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.445616576Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.445666136Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.465319882Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.465370673Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.484620351Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.484671072Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.503051325Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.503098376Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.521648852Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.521694982Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.540393961Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.540446242Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.558162793Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.558203363Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.577804038Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.577858919Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.596995625Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.597038706Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.617054508Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.617113329Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.635015404Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.635062874Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.653881925Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.653931506Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.671801959Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.67186087Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.672041014Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.672175946Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:47.765130568Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:47.765181969Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:47.7651909Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:47.76519764Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:47.76522226Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:47.76522679Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:47.76523274Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:47.76523794Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:47.765243101Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:47.76524955Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:47.765254701Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:47.765260961Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:47.765267191Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:47.765273421Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:47.765279861Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:47.765285151Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:47.765290751Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:47.765296571Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:47.765302702Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:47.765396213Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"5648a655-55d8-4eca-b45a-49def89502eb","remote_addr":"[fd00:1122:3344:101::1]:52853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\", TimeNotSynchronized), (\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\", TimeNotSynchronized), (\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\", TimeNotSynchronized), (\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\", TimeNotSynchronized), (\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\", TimeNotSynchronized), (\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\", TimeNotSynchronized), (\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\", TimeNotSynchronized), (\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\", TimeNotSynchronized), (\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\", TimeNotSynchronized), (\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\", TimeNotSynchronized), (\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\", TimeNotSynchronized), (\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\", TimeNotSynchronized), (\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\", TimeNotSynchronized), (\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\", TimeNotSynchronized), (\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\", TimeNotSynchronized), (\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\", TimeNotSynchronized), (\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\", TimeNotSynchronized), (\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\", TimeNotSynchronized), (\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\", TimeNotSynchronized)]","latency_us":477129,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:47.765629788Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"572.229538ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5648a655-55d8-4eca-b45a-49def89502eb\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:43:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5648a655-55d8-4eca-b45a-49def89502eb\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 57425cdc-844b-42e3-b45f-c8fb7871161f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b6a17b0f-0906-4eb9-94c3-528d000f68dc, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f4d5df0a-748d-40ab-8235-7c927aae50ce, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 23c9f9de-3736-4878-ba3b-118cebaacb82, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a684a100-6031-4b97-8de3-41e15037fbd5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 648e5f66-03f8-489e-a2bb-91cee8654a72, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 80d3c328-30b9-4b19-852b-d0002f908859, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 06241231-d15c-4adb-ae5b-e8e364e35528, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } } } }, OmicronZoneConfig { id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } } } }, OmicronZoneConfig { id: 57f3dc22-0a8a-40ec-a7b7-137531501134, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } } } }, OmicronZoneConfig { id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.338949685Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.339441534Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.339449824Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.360841351Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.360877671Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.37959033Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.379634611Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.401724281Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.401760922Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.422259653Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.422299863Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.44309978Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.4431322Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.463138562Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.463172513Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.483868078Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.483910468Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.504123054Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.504173625Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.525896628Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.525933919Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.547453668Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.547486249Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.570055908Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.570095878Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.591087048Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.591126439Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.612618008Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.612674169Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.634618456Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.634661587Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.65692358Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.656962461Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.67962333Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.679666071Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.700957506Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.700991127Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.722624758Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.722769421Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.744042236Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.744079857Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.765053236Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.765095297Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.765320651Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.765500385Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:48.863781963Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:48.863825034Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:48.863830064Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:48.863834754Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:48.863838844Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:48.863850035Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:48.863854055Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:48.863858055Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:48.863861915Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:48.863865965Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:48.863869905Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:48.863873585Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:48.863937696Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:48.863943226Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:48.863948886Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:48.863952676Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:48.863958227Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:48.863963477Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:48.863970087Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:48.86413689Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"a93892da-5a22-4ac4-adfb-0e1abf2ea0d1","remote_addr":"[fd00:1122:3344:101::1]:52853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\", TimeNotSynchronized), (\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\", TimeNotSynchronized), (\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\", TimeNotSynchronized), (\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\", TimeNotSynchronized), (\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\", TimeNotSynchronized), (\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\", TimeNotSynchronized), (\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\", TimeNotSynchronized), (\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\", TimeNotSynchronized), (\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\", TimeNotSynchronized), (\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\", TimeNotSynchronized), (\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\", TimeNotSynchronized), (\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\", TimeNotSynchronized), (\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\", TimeNotSynchronized), (\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\", TimeNotSynchronized), (\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\", TimeNotSynchronized), (\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\", TimeNotSynchronized), (\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\", TimeNotSynchronized), (\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\", TimeNotSynchronized), (\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\", TimeNotSynchronized)]","latency_us":524897,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:48.864376304Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"269.832933ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a93892da-5a22-4ac4-adfb-0e1abf2ea0d1\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:43:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a93892da-5a22-4ac4-adfb-0e1abf2ea0d1\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 57425cdc-844b-42e3-b45f-c8fb7871161f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b6a17b0f-0906-4eb9-94c3-528d000f68dc, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f4d5df0a-748d-40ab-8235-7c927aae50ce, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 23c9f9de-3736-4878-ba3b-118cebaacb82, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a684a100-6031-4b97-8de3-41e15037fbd5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 648e5f66-03f8-489e-a2bb-91cee8654a72, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 80d3c328-30b9-4b19-852b-d0002f908859, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 06241231-d15c-4adb-ae5b-e8e364e35528, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } } } }, OmicronZoneConfig { id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } } } }, OmicronZoneConfig { id: 57f3dc22-0a8a-40ec-a7b7-137531501134, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } } } }, OmicronZoneConfig { id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.134783029Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.135287098Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.135300858Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.157028881Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.157083302Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.176160498Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.176212868Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.195491737Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.195545558Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.211776602Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.211820953Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.230523371Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.230575952Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.248551868Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.248597889Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.26858557Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.268632061Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.287569364Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.287618795Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.305528219Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.30557604Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.325533201Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.325581652Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.346722785Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.346778546Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.364619939Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.36466791Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.383520831Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.383578142Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.404093993Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.404142164Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.422567138Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.422613948Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.440189537Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.440233377Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.461430151Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.461478992Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.481213949Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.48127632Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.501342203Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.501393254Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.520907058Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.520966699Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.521149342Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.521270624Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:49.620917808Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:49.620985609Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:49.620993589Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:49.621001079Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:49.621006099Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:49.621012359Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:49.62103898Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:49.62104597Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:49.62105209Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:49.62105638Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:49.62106259Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:49.62106857Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:49.621073031Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:49.621078871Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:49.621084871Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:49.621092651Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:49.621100571Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:49.621107851Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:49.621113611Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:49.621254014Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"53c74a00-1dba-46ba-b84f-612e0f4f5094","remote_addr":"[fd00:1122:3344:101::1]:52853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\", TimeNotSynchronized), (\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\", TimeNotSynchronized), (\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\", TimeNotSynchronized), (\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\", TimeNotSynchronized), (\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\", TimeNotSynchronized), (\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\", TimeNotSynchronized), (\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\", TimeNotSynchronized), (\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\", TimeNotSynchronized), (\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\", TimeNotSynchronized), (\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\", TimeNotSynchronized), (\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\", TimeNotSynchronized), (\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\", TimeNotSynchronized), (\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\", TimeNotSynchronized), (\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\", TimeNotSynchronized), (\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\", TimeNotSynchronized), (\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\", TimeNotSynchronized), (\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\", TimeNotSynchronized), (\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\", TimeNotSynchronized), (\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\", TimeNotSynchronized)]","latency_us":486166,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:49.621491438Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"502.093187ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"53c74a00-1dba-46ba-b84f-612e0f4f5094\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:43:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"53c74a00-1dba-46ba-b84f-612e0f4f5094\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 57425cdc-844b-42e3-b45f-c8fb7871161f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b6a17b0f-0906-4eb9-94c3-528d000f68dc, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f4d5df0a-748d-40ab-8235-7c927aae50ce, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 23c9f9de-3736-4878-ba3b-118cebaacb82, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a684a100-6031-4b97-8de3-41e15037fbd5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 648e5f66-03f8-489e-a2bb-91cee8654a72, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 80d3c328-30b9-4b19-852b-d0002f908859, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 06241231-d15c-4adb-ae5b-e8e364e35528, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } } } }, OmicronZoneConfig { id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } } } }, OmicronZoneConfig { id: 57f3dc22-0a8a-40ec-a7b7-137531501134, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } } } }, OmicronZoneConfig { id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.124785277Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.125258656Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.125267206Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.14812002Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.148174031Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.168214104Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.168265655Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.187635595Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.187682196Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.206836913Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.206895674Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.227075159Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.22712536Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.246976899Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.24702658Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.266536143Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.266589004Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.286571756Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.286619387Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.307407423Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.307466094Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.328658048Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.328712649Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.349158669Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.34920766Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.369217892Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.369266643Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.389220934Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.389275425Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.410888676Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.410937277Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.429949281Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.429994382Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.450421482Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.450477193Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.47020124Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.47025371Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.490087409Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.490260173Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.509818717Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.509868018Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.528751409Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.52880573Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.529015144Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.529145767Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:50.624831619Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:50.624891609Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:50.6248996Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:50.62490522Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:50.62491069Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:50.62491768Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:50.62492328Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:50.62493834Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:50.62494533Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:50.62495087Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:50.624957671Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:50.624963541Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:50.624967801Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:50.624971651Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:50.624977051Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:50.624981151Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:50.624984751Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:50.624988681Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:50.624992511Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:50.625071593Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"d0b16590-96ff-4292-9400-f83aaee4626f","remote_addr":"[fd00:1122:3344:101::1]:52853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\", TimeNotSynchronized), (\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\", TimeNotSynchronized), (\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\", TimeNotSynchronized), (\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\", TimeNotSynchronized), (\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\", TimeNotSynchronized), (\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\", TimeNotSynchronized), (\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\", TimeNotSynchronized), (\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\", TimeNotSynchronized), (\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\", TimeNotSynchronized), (\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\", TimeNotSynchronized), (\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\", TimeNotSynchronized), (\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\", TimeNotSynchronized), (\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\", TimeNotSynchronized), (\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\", TimeNotSynchronized), (\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\", TimeNotSynchronized), (\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\", TimeNotSynchronized), (\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\", TimeNotSynchronized), (\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\", TimeNotSynchronized), (\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\", TimeNotSynchronized)]","latency_us":500003,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:50.625291147Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"492.655909ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d0b16590-96ff-4292-9400-f83aaee4626f\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:43:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d0b16590-96ff-4292-9400-f83aaee4626f\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 57425cdc-844b-42e3-b45f-c8fb7871161f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b6a17b0f-0906-4eb9-94c3-528d000f68dc, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f4d5df0a-748d-40ab-8235-7c927aae50ce, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 23c9f9de-3736-4878-ba3b-118cebaacb82, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a684a100-6031-4b97-8de3-41e15037fbd5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 648e5f66-03f8-489e-a2bb-91cee8654a72, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 80d3c328-30b9-4b19-852b-d0002f908859, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 06241231-d15c-4adb-ae5b-e8e364e35528, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } } } }, OmicronZoneConfig { id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } } } }, OmicronZoneConfig { id: 57f3dc22-0a8a-40ec-a7b7-137531501134, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } } } }, OmicronZoneConfig { id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.11883108Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.119319319Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.119335259Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.141221635Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.141273156Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.162299987Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.162349677Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.181179388Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.181226699Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.201791701Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.201839032Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.221976407Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.222027418Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.241572061Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.241621002Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.261797248Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.261846558Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.281633667Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.281680127Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.299104403Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.299151294Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.318578785Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.318632766Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.33873285Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.338774441Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.359241821Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.359289252Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.38237057Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.382418281Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.401289902Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.401349913Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.421919556Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.421967316Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.441299517Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.441352287Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.460687217Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.46085868Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.480504726Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.480553947Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.500341285Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.500389596Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.520348167Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.520408508Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.520593082Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.520712174Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:51.615686923Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:51.615748584Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:51.615754174Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:51.615761674Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:51.615765814Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:51.615771524Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:51.615778844Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:51.615784615Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:51.615791975Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:51.615815755Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:51.615820185Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:51.615827256Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:51.615832665Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:51.615838086Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:51.615843206Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:51.615849996Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:51.615855526Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:51.615859196Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:51.615862706Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:51.615954588Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"71e088bc-0239-4d68-bd9c-7d33dc9720f8","remote_addr":"[fd00:1122:3344:101::1]:52853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\", TimeNotSynchronized), (\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\", TimeNotSynchronized), (\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\", TimeNotSynchronized), (\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\", TimeNotSynchronized), (\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\", TimeNotSynchronized), (\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\", TimeNotSynchronized), (\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\", TimeNotSynchronized), (\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\", TimeNotSynchronized), (\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\", TimeNotSynchronized), (\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\", TimeNotSynchronized), (\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\", TimeNotSynchronized), (\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\", TimeNotSynchronized), (\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\", TimeNotSynchronized), (\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\", TimeNotSynchronized), (\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\", TimeNotSynchronized), (\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\", TimeNotSynchronized), (\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\", TimeNotSynchronized), (\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\", TimeNotSynchronized), (\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\", TimeNotSynchronized)]","latency_us":496849,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:51.616194982Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"673.639551ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"71e088bc-0239-4d68-bd9c-7d33dc9720f8\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:43:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"71e088bc-0239-4d68-bd9c-7d33dc9720f8\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 57425cdc-844b-42e3-b45f-c8fb7871161f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b6a17b0f-0906-4eb9-94c3-528d000f68dc, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f4d5df0a-748d-40ab-8235-7c927aae50ce, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 23c9f9de-3736-4878-ba3b-118cebaacb82, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a684a100-6031-4b97-8de3-41e15037fbd5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 648e5f66-03f8-489e-a2bb-91cee8654a72, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 80d3c328-30b9-4b19-852b-d0002f908859, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 06241231-d15c-4adb-ae5b-e8e364e35528, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } } } }, OmicronZoneConfig { id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } } } }, OmicronZoneConfig { id: 57f3dc22-0a8a-40ec-a7b7-137531501134, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } } } }, OmicronZoneConfig { id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.291123298Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.291638297Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.291655098Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.314725575Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.314787807Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.335898229Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.335953319Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.354079118Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.354131539Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.375558036Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.375614857Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.394509499Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.394575451Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.413222118Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.413272959Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.43154074Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.431596181Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.449599056Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.449653188Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.468729753Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.468778364Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.488246136Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.488289847Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.509084173Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.509136494Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.530207966Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.530244846Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.54870481Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.548750251Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.569531927Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.569565968Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.587789198Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.587825389Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.604867967Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.604905238Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.623929002Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.623968603Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.645774807Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.645816508Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.664221521Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.664272092Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.683605712Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.683652183Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.683870657Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.684026119Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:52.784576639Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:52.784637081Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:52.784643491Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:52.784650731Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:52.784656891Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:52.784662531Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:52.784666351Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:52.784670231Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:52.784674231Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:52.784678371Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:52.784689332Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:52.784693332Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:52.784697342Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:52.784702432Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:52.784708232Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:52.784714022Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:52.784811934Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:52.784818894Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:52.784824144Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:52.784979517Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"a2fad64f-da37-483e-b393-7cdc82e953ac","remote_addr":"[fd00:1122:3344:101::1]:52853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\", TimeNotSynchronized), (\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\", TimeNotSynchronized), (\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\", TimeNotSynchronized), (\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\", TimeNotSynchronized), (\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\", TimeNotSynchronized), (\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\", TimeNotSynchronized), (\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\", TimeNotSynchronized), (\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\", TimeNotSynchronized), (\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\", TimeNotSynchronized), (\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\", TimeNotSynchronized), (\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\", TimeNotSynchronized), (\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\", TimeNotSynchronized), (\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\", TimeNotSynchronized), (\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\", TimeNotSynchronized), (\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\", TimeNotSynchronized), (\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\", TimeNotSynchronized), (\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\", TimeNotSynchronized), (\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\", TimeNotSynchronized), (\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\", TimeNotSynchronized)]","latency_us":493554,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:52.785231041Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"1.326579596s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a2fad64f-da37-483e-b393-7cdc82e953ac\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:43:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a2fad64f-da37-483e-b393-7cdc82e953ac\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 57425cdc-844b-42e3-b45f-c8fb7871161f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b6a17b0f-0906-4eb9-94c3-528d000f68dc, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f4d5df0a-748d-40ab-8235-7c927aae50ce, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 23c9f9de-3736-4878-ba3b-118cebaacb82, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a684a100-6031-4b97-8de3-41e15037fbd5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 648e5f66-03f8-489e-a2bb-91cee8654a72, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 80d3c328-30b9-4b19-852b-d0002f908859, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 06241231-d15c-4adb-ae5b-e8e364e35528, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } } } }, OmicronZoneConfig { id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } } } }, OmicronZoneConfig { id: 57f3dc22-0a8a-40ec-a7b7-137531501134, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } } } }, OmicronZoneConfig { id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.112742119Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.113263909Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.113278049Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.134834229Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.13487434Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.156811657Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.156855088Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.177553232Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.177597753Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.195841814Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.195880214Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.212524076Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.212567636Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.231346366Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.231384067Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.249620527Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.249660498Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.271398071Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.271432822Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.291541466Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.291573256Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.310905116Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.310945087Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.327340044Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.327380194Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.346880297Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.346915628Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.36526171Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.365299131Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.383011961Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.383047522Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.402077816Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.402112937Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.421778523Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.421816164Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.44537196Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.445882269Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.466462042Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.466497933Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.487486382Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.487524713Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.509054353Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.509094353Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.509316268Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.509598762Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:54.609545002Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:54.609602913Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:54.609611333Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:54.609622843Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:54.609628713Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:54.609634163Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:54.609639593Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:54.609645613Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:54.609651104Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:54.609656884Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:54.609663734Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:54.609685224Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:54.609690884Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:54.609696494Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:54.609701894Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:54.609708554Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:54.609713965Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:54.609717995Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:54.609721475Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:54.609811326Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"938cfe04-82d9-4813-b7d1-cc13615a62fc","remote_addr":"[fd00:1122:3344:101::1]:52853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\", TimeNotSynchronized), (\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\", TimeNotSynchronized), (\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\", TimeNotSynchronized), (\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\", TimeNotSynchronized), (\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\", TimeNotSynchronized), (\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\", TimeNotSynchronized), (\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\", TimeNotSynchronized), (\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\", TimeNotSynchronized), (\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\", TimeNotSynchronized), (\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\", TimeNotSynchronized), (\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\", TimeNotSynchronized), (\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\", TimeNotSynchronized), (\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\", TimeNotSynchronized), (\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\", TimeNotSynchronized), (\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\", TimeNotSynchronized), (\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\", TimeNotSynchronized), (\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\", TimeNotSynchronized), (\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\", TimeNotSynchronized), (\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\", TimeNotSynchronized)]","latency_us":496766,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:54.61003452Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"1.033950406s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"938cfe04-82d9-4813-b7d1-cc13615a62fc\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:43:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"938cfe04-82d9-4813-b7d1-cc13615a62fc\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 57425cdc-844b-42e3-b45f-c8fb7871161f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b6a17b0f-0906-4eb9-94c3-528d000f68dc, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f4d5df0a-748d-40ab-8235-7c927aae50ce, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 23c9f9de-3736-4878-ba3b-118cebaacb82, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a684a100-6031-4b97-8de3-41e15037fbd5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 648e5f66-03f8-489e-a2bb-91cee8654a72, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 80d3c328-30b9-4b19-852b-d0002f908859, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 06241231-d15c-4adb-ae5b-e8e364e35528, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } } } }, OmicronZoneConfig { id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } } } }, OmicronZoneConfig { id: 57f3dc22-0a8a-40ec-a7b7-137531501134, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } } } }, OmicronZoneConfig { id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.644687128Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.645167016Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.645174296Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.668241703Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.668302234Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.690650499Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.69070347Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.711262112Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.711309163Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.732335203Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.732387354Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.751893127Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.751957468Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.772375428Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.772435049Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.793428369Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.79348291Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.811414304Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.811469676Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.83215728Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.832203151Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.850633414Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.850691126Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.87027639Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.870326351Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.889413536Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.889457637Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.91005911Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.910120841Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.931443487Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.931499988Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.949897361Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.949958912Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.970364112Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.970413042Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.988956928Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:55.989002889Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:56.009595482Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:56.009648312Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:56.027873502Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:56.027918803Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:56.047290514Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:56.047370575Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:56.047533548Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:56.04765314Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:56.138708108Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:56.138758669Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:56.138763589Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:56.13877127Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:56.13877678Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:56.13878355Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:56.13879021Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:56.13879718Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:56.13880309Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:56.13880834Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:56.138815201Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:56.138821011Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:56.138837881Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:56.138843491Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:56.138849121Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:56.138854421Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:56.138861301Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:56.138865211Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:56.138869091Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:56.138958503Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"5d914cec-a688-416c-82b9-557e38538c33","remote_addr":"[fd00:1122:3344:101::1]:52853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\", TimeNotSynchronized), (\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\", TimeNotSynchronized), (\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\", TimeNotSynchronized), (\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\", TimeNotSynchronized), (\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\", TimeNotSynchronized), (\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\", TimeNotSynchronized), (\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\", TimeNotSynchronized), (\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\", TimeNotSynchronized), (\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\", TimeNotSynchronized), (\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\", TimeNotSynchronized), (\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\", TimeNotSynchronized), (\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\", TimeNotSynchronized), (\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\", TimeNotSynchronized), (\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\", TimeNotSynchronized), (\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\", TimeNotSynchronized), (\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\", TimeNotSynchronized), (\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\", TimeNotSynchronized), (\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\", TimeNotSynchronized), (\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\", TimeNotSynchronized)]","latency_us":493977,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:56.139178557Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"2.10162743s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5d914cec-a688-416c-82b9-557e38538c33\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:43:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5d914cec-a688-416c-82b9-557e38538c33\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 57425cdc-844b-42e3-b45f-c8fb7871161f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b6a17b0f-0906-4eb9-94c3-528d000f68dc, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f4d5df0a-748d-40ab-8235-7c927aae50ce, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 23c9f9de-3736-4878-ba3b-118cebaacb82, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a684a100-6031-4b97-8de3-41e15037fbd5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 648e5f66-03f8-489e-a2bb-91cee8654a72, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 80d3c328-30b9-4b19-852b-d0002f908859, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 06241231-d15c-4adb-ae5b-e8e364e35528, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } } } }, OmicronZoneConfig { id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } } } }, OmicronZoneConfig { id: 57f3dc22-0a8a-40ec-a7b7-137531501134, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } } } }, OmicronZoneConfig { id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.242320064Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.243089898Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.243102858Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.265556884Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.265608025Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.28520939Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.285250011Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.304064451Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.304107192Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.321987396Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.322024017Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.340046853Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.340096833Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.358470726Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.358521567Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.375670297Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.375720978Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.393669713Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.393703614Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.411489686Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.411526166Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.42995129Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.429996611Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.449119727Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.449168198Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.466798947Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.466855838Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.485166879Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.48521219Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.505687451Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.505736812Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.525471309Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.52551534Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.544497063Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.544540744Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.564843181Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.564889442Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.585312462Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.585356493Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.605386865Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.605436786Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.624104424Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.624156035Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.624326288Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.62444822Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:58.715409727Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:58.715470388Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:58.715480468Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:58.715487378Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:58.715495828Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:58.715504048Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:58.715510608Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:58.715517488Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:58.715524359Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:58.715532289Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:58.715538779Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:58.715547289Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:58.715555509Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:58.71558131Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:58.71558989Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:58.7155978Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:58.71560571Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:58.71561063Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:58.71561673Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:58.715701342Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"8eddc4ca-05b8-416b-9ea7-28d8829a2592","remote_addr":"[fd00:1122:3344:101::1]:52853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\", TimeNotSynchronized), (\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\", TimeNotSynchronized), (\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\", TimeNotSynchronized), (\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\", TimeNotSynchronized), (\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\", TimeNotSynchronized), (\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\", TimeNotSynchronized), (\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\", TimeNotSynchronized), (\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\", TimeNotSynchronized), (\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\", TimeNotSynchronized), (\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\", TimeNotSynchronized), (\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\", TimeNotSynchronized), (\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\", TimeNotSynchronized), (\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\", TimeNotSynchronized), (\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\", TimeNotSynchronized), (\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\", TimeNotSynchronized), (\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\", TimeNotSynchronized), (\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\", TimeNotSynchronized), (\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\", TimeNotSynchronized), (\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\", TimeNotSynchronized)]","latency_us":472952,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:43:58.715925856Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"1.20955001s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8eddc4ca-05b8-416b-9ea7-28d8829a2592\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:43:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8eddc4ca-05b8-416b-9ea7-28d8829a2592\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 57425cdc-844b-42e3-b45f-c8fb7871161f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b6a17b0f-0906-4eb9-94c3-528d000f68dc, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f4d5df0a-748d-40ab-8235-7c927aae50ce, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 23c9f9de-3736-4878-ba3b-118cebaacb82, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a684a100-6031-4b97-8de3-41e15037fbd5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 648e5f66-03f8-489e-a2bb-91cee8654a72, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 80d3c328-30b9-4b19-852b-d0002f908859, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 06241231-d15c-4adb-ae5b-e8e364e35528, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } } } }, OmicronZoneConfig { id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } } } }, OmicronZoneConfig { id: 57f3dc22-0a8a-40ec-a7b7-137531501134, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } } } }, OmicronZoneConfig { id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:59.926599878Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:59.927130128Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:59.927142128Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:59.948133548Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:59.948201109Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:59.968182971Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:59.968234602Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:59.98638671Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:43:59.986425501Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.004376316Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.004417997Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.022945212Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.022986753Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.040540261Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.040589012Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.057936316Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.057979606Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.075974952Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.076029183Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.093101612Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.093153973Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.111086948Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.111136699Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.128265549Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.1283233Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.145222025Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.145270536Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.163498616Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.163541147Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.181048814Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.181098825Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.198725134Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.198772475Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.21616873Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.216219001Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.235790475Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.235830445Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.254258919Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.25431961Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.271766646Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.271809787Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.29021236Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.290271551Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.290491815Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.290624867Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:00.384532627Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:00.384583718Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:00.384591488Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:00.384598028Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:00.384604638Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:00.384610018Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:00.384616609Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:00.384622889Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:00.384629379Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:00.384634629Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:00.384640809Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:00.384644949Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:00.384651169Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:00.384656489Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:00.384661849Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:00.38467872Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:00.38468512Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:00.3846903Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:00.38469622Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:00.384813472Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"5ab38bb7-691d-4ad0-bedb-21fb7ceb1c58","remote_addr":"[fd00:1122:3344:101::1]:52853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\", TimeNotSynchronized), (\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\", TimeNotSynchronized), (\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\", TimeNotSynchronized), (\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\", TimeNotSynchronized), (\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\", TimeNotSynchronized), (\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\", TimeNotSynchronized), (\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\", TimeNotSynchronized), (\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\", TimeNotSynchronized), (\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\", TimeNotSynchronized), (\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\", TimeNotSynchronized), (\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\", TimeNotSynchronized), (\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\", TimeNotSynchronized), (\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\", TimeNotSynchronized), (\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\", TimeNotSynchronized), (\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\", TimeNotSynchronized), (\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\", TimeNotSynchronized), (\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\", TimeNotSynchronized), (\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\", TimeNotSynchronized), (\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\", TimeNotSynchronized)]","latency_us":457910,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:00.385050506Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"1.691299746s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5ab38bb7-691d-4ad0-bedb-21fb7ceb1c58\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:44:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5ab38bb7-691d-4ad0-bedb-21fb7ceb1c58\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 57425cdc-844b-42e3-b45f-c8fb7871161f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b6a17b0f-0906-4eb9-94c3-528d000f68dc, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f4d5df0a-748d-40ab-8235-7c927aae50ce, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 23c9f9de-3736-4878-ba3b-118cebaacb82, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a684a100-6031-4b97-8de3-41e15037fbd5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 648e5f66-03f8-489e-a2bb-91cee8654a72, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 80d3c328-30b9-4b19-852b-d0002f908859, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 06241231-d15c-4adb-ae5b-e8e364e35528, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } } } }, OmicronZoneConfig { id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } } } }, OmicronZoneConfig { id: 57f3dc22-0a8a-40ec-a7b7-137531501134, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } } } }, OmicronZoneConfig { id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.077174113Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.077634352Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.077643242Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.099181792Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.099235513Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.118834288Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.118880779Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.137107688Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.137151209Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.156642042Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.156684783Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.174112858Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.174161439Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.191791138Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.191843459Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.209798634Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.209847525Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.227402883Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.227458664Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.245114753Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.245163794Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.264573116Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.264608256Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.283985907Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.284035618Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.302161706Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.302204437Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.320973496Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.321012337Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.339163766Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.339204676Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.357454347Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.357508088Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.375476793Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.375529654Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.393636252Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.393668062Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.412035795Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.412072685Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.430422807Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.430455498Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.44822541Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.448267881Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.448520025Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.448675778Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:02.541926946Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:02.541984227Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:02.541991517Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:02.541998337Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:02.542003317Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:02.542008697Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:02.542014607Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:02.542021087Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:02.542027207Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:02.542033378Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:02.542039108Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:02.542044568Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:02.542051718Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:02.542057198Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:02.542063438Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:02.542069148Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:02.542092199Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:02.542098469Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:02.542102669Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:02.542199111Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"b23a63be-d6c6-4537-a070-1ad126ee5856","remote_addr":"[fd00:1122:3344:101::1]:52853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\", TimeNotSynchronized), (\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\", TimeNotSynchronized), (\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\", TimeNotSynchronized), (\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\", TimeNotSynchronized), (\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\", TimeNotSynchronized), (\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\", TimeNotSynchronized), (\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\", TimeNotSynchronized), (\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\", TimeNotSynchronized), (\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\", TimeNotSynchronized), (\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\", TimeNotSynchronized), (\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\", TimeNotSynchronized), (\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\", TimeNotSynchronized), (\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\", TimeNotSynchronized), (\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\", TimeNotSynchronized), (\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\", TimeNotSynchronized), (\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\", TimeNotSynchronized), (\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\", TimeNotSynchronized), (\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\", TimeNotSynchronized), (\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\", TimeNotSynchronized)]","latency_us":464743,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:02.542445585Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"3.46100635s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b23a63be-d6c6-4537-a070-1ad126ee5856\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:44:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b23a63be-d6c6-4537-a070-1ad126ee5856\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 57425cdc-844b-42e3-b45f-c8fb7871161f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b6a17b0f-0906-4eb9-94c3-528d000f68dc, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f4d5df0a-748d-40ab-8235-7c927aae50ce, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 23c9f9de-3736-4878-ba3b-118cebaacb82, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a684a100-6031-4b97-8de3-41e15037fbd5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 648e5f66-03f8-489e-a2bb-91cee8654a72, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 80d3c328-30b9-4b19-852b-d0002f908859, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 06241231-d15c-4adb-ae5b-e8e364e35528, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } } } }, OmicronZoneConfig { id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } } } }, OmicronZoneConfig { id: 57f3dc22-0a8a-40ec-a7b7-137531501134, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } } } }, OmicronZoneConfig { id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.005430884Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.005936313Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.005975104Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.02898533Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.029072312Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.050144213Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.050206564Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.069373982Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.069440393Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.088432356Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.088505948Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.108732154Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.108790945Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.12732818Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.127386032Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.146178961Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.146242153Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.165900378Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.165952779Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.184526086Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.184588627Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.203510889Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.203582431Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.222374151Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.222437452Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.240772244Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.240826615Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.259059665Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.259127516Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.277409177Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.277461028Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.29526575Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.295313981Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.312292028Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.31235864Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.330458747Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.330514768Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.34772624Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.34992614Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.365581323Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.367815233Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.38256218Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.382621481Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.382830595Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.382973228Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:06.473567627Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:06.473638769Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:06.473663829Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:06.473681329Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:06.47369056Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:06.47369915Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:06.47372093Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:06.47372795Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:06.473743021Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:06.473750071Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:06.473767571Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:06.473773011Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:06.473779481Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:06.473785151Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:06.473790401Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:06.473795471Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:06.473800662Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:06.473813002Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:06.473822052Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:06.473889543Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"f8539a9a-ae0a-43dd-8a7c-0ed2ac3dc282","remote_addr":"[fd00:1122:3344:101::1]:52853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\", TimeNotSynchronized), (\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\", TimeNotSynchronized), (\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\", TimeNotSynchronized), (\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\", TimeNotSynchronized), (\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\", TimeNotSynchronized), (\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\", TimeNotSynchronized), (\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\", TimeNotSynchronized), (\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\", TimeNotSynchronized), (\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\", TimeNotSynchronized), (\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\", TimeNotSynchronized), (\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\", TimeNotSynchronized), (\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\", TimeNotSynchronized), (\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\", TimeNotSynchronized), (\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\", TimeNotSynchronized), (\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\", TimeNotSynchronized), (\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\", TimeNotSynchronized), (\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\", TimeNotSynchronized), (\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\", TimeNotSynchronized), (\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\", TimeNotSynchronized)]","latency_us":468141,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:06.474144878Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"1.843988878s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f8539a9a-ae0a-43dd-8a7c-0ed2ac3dc282\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:44:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f8539a9a-ae0a-43dd-8a7c-0ed2ac3dc282\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 57425cdc-844b-42e3-b45f-c8fb7871161f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b6a17b0f-0906-4eb9-94c3-528d000f68dc, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f4d5df0a-748d-40ab-8235-7c927aae50ce, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 23c9f9de-3736-4878-ba3b-118cebaacb82, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a684a100-6031-4b97-8de3-41e15037fbd5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 648e5f66-03f8-489e-a2bb-91cee8654a72, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 80d3c328-30b9-4b19-852b-d0002f908859, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 06241231-d15c-4adb-ae5b-e8e364e35528, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } } } }, OmicronZoneConfig { id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } } } }, OmicronZoneConfig { id: 57f3dc22-0a8a-40ec-a7b7-137531501134, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } } } }, OmicronZoneConfig { id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.315380319Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.315945039Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.315958729Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.337693204Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.337758075Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.356501505Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.356552366Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.375059662Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.375108533Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.393177351Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.393224983Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.411149697Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.411220348Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.429133723Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.429190414Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.447082169Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.44712723Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.464292201Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.464348392Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.482476161Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.482526072Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.500809394Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.500846575Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.520121365Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.520175985Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.53858185Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.53863314Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.556170309Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.55621509Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.574383139Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.574466641Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.592683456Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.592734362Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.61020281Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.610262271Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.629715824Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.631971265Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.647828152Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.647872863Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.664856371Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.664905272Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.681676227Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.681725788Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.681968092Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.682109175Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:08.769355478Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:08.769430009Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:08.76944138Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:08.76946629Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:08.76947684Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:08.76948497Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:08.769495161Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:08.769505561Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:08.769513521Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:08.769523861Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:08.769533851Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:08.769543601Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:08.769561552Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:08.769571292Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:08.769577262Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:08.769590852Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:08.769596942Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:08.769603252Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:08.769622593Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:08.769696414Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"3aa678d7-35dc-48b9-b625-fb7df19ede3c","remote_addr":"[fd00:1122:3344:101::1]:52853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\", TimeNotSynchronized), (\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\", TimeNotSynchronized), (\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\", TimeNotSynchronized), (\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\", TimeNotSynchronized), (\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\", TimeNotSynchronized), (\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\", TimeNotSynchronized), (\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\", TimeNotSynchronized), (\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\", TimeNotSynchronized), (\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\", TimeNotSynchronized), (\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\", TimeNotSynchronized), (\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\", TimeNotSynchronized), (\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\", TimeNotSynchronized), (\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\", TimeNotSynchronized), (\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\", TimeNotSynchronized), (\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\", TimeNotSynchronized), (\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\", TimeNotSynchronized), (\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\", TimeNotSynchronized), (\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\", TimeNotSynchronized), (\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\", TimeNotSynchronized)]","latency_us":455217,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:08.769925398Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"5.408723479s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3aa678d7-35dc-48b9-b625-fb7df19ede3c\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:44:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3aa678d7-35dc-48b9-b625-fb7df19ede3c\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:12.605996229Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37758"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:12.606089861Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"86d9c9d9-585b-4cf5-8475-3bc3ddb9696f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37758","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 57425cdc-844b-42e3-b45f-c8fb7871161f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b6a17b0f-0906-4eb9-94c3-528d000f68dc, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f4d5df0a-748d-40ab-8235-7c927aae50ce, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 23c9f9de-3736-4878-ba3b-118cebaacb82, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a684a100-6031-4b97-8de3-41e15037fbd5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 648e5f66-03f8-489e-a2bb-91cee8654a72, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 80d3c328-30b9-4b19-852b-d0002f908859, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 06241231-d15c-4adb-ae5b-e8e364e35528, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } } } }, OmicronZoneConfig { id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } } } }, OmicronZoneConfig { id: 57f3dc22-0a8a-40ec-a7b7-137531501134, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } } } }, OmicronZoneConfig { id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.165749446Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.167215613Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.167230303Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.187677514Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.187736555Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.207136277Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.209270596Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.224800658Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.224854559Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.242895526Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.242947147Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.261330251Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.261385102Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.27947915Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.279550402Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.296629452Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.296680873Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.313268704Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.313313155Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.330915034Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.333047333Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.348272129Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.34831995Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.366433249Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.368581898Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.384188361Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.384257132Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.401857982Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.401917767Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.419664995Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.419720326Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.437786224Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.439985564Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.455687869Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.455742192Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.473207086Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.473261087Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.490165594Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.490225446Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.507858475Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.507911856Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.52575578Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.525839232Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.526056086Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.526200559Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:14.614823817Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:14.614916959Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:14.614927219Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:14.614935819Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:14.614941369Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:14.61496137Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:14.61497864Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:14.61498699Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:14.61499397Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:14.61500213Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:14.61501015Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:14.615023Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:14.615031181Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:14.615039231Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:14.615050541Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:14.615057531Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:14.615071401Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:14.615077052Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:14.615082222Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:14.615160653Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"bc68910c-55e4-4eff-a3bb-dfdb8a3a2fd5","remote_addr":"[fd00:1122:3344:101::1]:52853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\", TimeNotSynchronized), (\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\", TimeNotSynchronized), (\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\", TimeNotSynchronized), (\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\", TimeNotSynchronized), (\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\", TimeNotSynchronized), (\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\", TimeNotSynchronized), (\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\", TimeNotSynchronized), (\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\", TimeNotSynchronized), (\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\", TimeNotSynchronized), (\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\", TimeNotSynchronized), (\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\", TimeNotSynchronized), (\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\", TimeNotSynchronized), (\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\", TimeNotSynchronized), (\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\", TimeNotSynchronized), (\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\", TimeNotSynchronized), (\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\", TimeNotSynchronized), (\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\", TimeNotSynchronized), (\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\", TimeNotSynchronized), (\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\", TimeNotSynchronized)]","latency_us":450206,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:14.615360047Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"4.034242589s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bc68910c-55e4-4eff-a3bb-dfdb8a3a2fd5\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:44:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bc68910c-55e4-4eff-a3bb-dfdb8a3a2fd5\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 57425cdc-844b-42e3-b45f-c8fb7871161f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b6a17b0f-0906-4eb9-94c3-528d000f68dc, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f4d5df0a-748d-40ab-8235-7c927aae50ce, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 23c9f9de-3736-4878-ba3b-118cebaacb82, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a684a100-6031-4b97-8de3-41e15037fbd5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 648e5f66-03f8-489e-a2bb-91cee8654a72, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 80d3c328-30b9-4b19-852b-d0002f908859, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 06241231-d15c-4adb-ae5b-e8e364e35528, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } } } }, OmicronZoneConfig { id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } } } }, OmicronZoneConfig { id: 57f3dc22-0a8a-40ec-a7b7-137531501134, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } } } }, OmicronZoneConfig { id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.639020434Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.639542763Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.639550963Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.660818869Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.66086477Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.680317223Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.680373064Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.699636744Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.699682125Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.718032048Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.718075599Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.736542484Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.736587295Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.755179712Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.755215553Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.772800572Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.77286003Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.790110066Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.790148927Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.808594691Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.808630252Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.826421875Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.826476206Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.844067335Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.844108856Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.863657971Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.863701822Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.881835081Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.881891442Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.900292926Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.900334616Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.919517165Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.919559845Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.938039971Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.938079792Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.956517926Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.956555527Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.974668346Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.974706816Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.99362434Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:18.99366034Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:19.012043894Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:19.012084295Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:19.012331329Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:19.012466972Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:19.105635573Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:19.105714504Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:19.105736515Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:19.105745875Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:19.105755785Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:19.105765725Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:19.105775525Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:19.105783555Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:19.105793536Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:19.105801096Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:19.105811276Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:19.105842326Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:19.105856597Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:19.105867767Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:19.105874077Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:19.105883617Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:19.105891427Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:19.105916828Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:19.105924054Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:19.105971099Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"db34b5ee-7b53-44b7-8ac0-f07eda92f9b4","remote_addr":"[fd00:1122:3344:101::1]:52853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\", TimeNotSynchronized), (\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\", TimeNotSynchronized), (\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\", TimeNotSynchronized), (\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\", TimeNotSynchronized), (\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\", TimeNotSynchronized), (\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\", TimeNotSynchronized), (\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\", TimeNotSynchronized), (\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\", TimeNotSynchronized), (\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\", TimeNotSynchronized), (\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\", TimeNotSynchronized), (\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\", TimeNotSynchronized), (\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\", TimeNotSynchronized), (\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\", TimeNotSynchronized), (\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\", TimeNotSynchronized), (\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\", TimeNotSynchronized), (\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\", TimeNotSynchronized), (\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\", TimeNotSynchronized), (\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\", TimeNotSynchronized), (\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\", TimeNotSynchronized)]","latency_us":467934,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:19.106185333Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"6.79230886s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"db34b5ee-7b53-44b7-8ac0-f07eda92f9b4\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:44:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"db34b5ee-7b53-44b7-8ac0-f07eda92f9b4\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: c037c9e2-43a2-432a-bcd7-352e08f26490, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 655db966-87f1-48aa-8868-000f036af1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a01b5bfc-3390-497f-96d2-052158f09a46, kind: Service { id: 2168e6f2-fbb0-4d6d-8811-9fd75847a2a7 }, name: Name(\"ntp-2168e6f2-fbb0-4d6d-8811-9fd75847a2a7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 167])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7311d36-617f-4f72-be7a-3427dc69c306, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f597bf55-7c6a-456c-8e41-286fd5037349, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 57425cdc-844b-42e3-b45f-c8fb7871161f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b6a17b0f-0906-4eb9-94c3-528d000f68dc, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: f4d5df0a-748d-40ab-8235-7c927aae50ce, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 23c9f9de-3736-4878-ba3b-118cebaacb82, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a684a100-6031-4b97-8de3-41e15037fbd5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 648e5f66-03f8-489e-a2bb-91cee8654a72, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External } } } }, OmicronZoneConfig { id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External } } } }, OmicronZoneConfig { id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External } } } }, OmicronZoneConfig { id: 80d3c328-30b9-4b19-852b-d0002f908859, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External } } } }, OmicronZoneConfig { id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External } } } }, OmicronZoneConfig { id: 06241231-d15c-4adb-ae5b-e8e364e35528, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External } } } }, OmicronZoneConfig { id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External } } } }, OmicronZoneConfig { id: 57f3dc22-0a8a-40ec-a7b7-137531501134, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External } } } }, OmicronZoneConfig { id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:25.881670635Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:25.882807916Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:25.882815616Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 55e16fed-0b39-4c9e-94aa-dbf1dfed9795, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:25.906663409Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:25.90670913Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c037c9e2-43a2-432a-bcd7-352e08f26490, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:25.92655877Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:25.926607621Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 655db966-87f1-48aa-8868-000f036af1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:25.945500414Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:25.945557115Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7311d36-617f-4f72-be7a-3427dc69c306, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:25.965798902Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:25.965849223Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f597bf55-7c6a-456c-8e41-286fd5037349, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:25.983648006Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:25.983703817Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ddf7c878-622c-4bc8-8b0b-5a158ff3b198, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.001686923Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.001736964Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9d0139eb-7e07-43fa-92d7-f8a6676bf465, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.019584698Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.019633599Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f7108ea5-965b-4e70-a9a0-7ffd5d13885f, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.03680916Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.036856812Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57425cdc-844b-42e3-b45f-c8fb7871161f, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.05492977Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.054969101Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.074943203Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.074988334Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8bf3945e-e049-4e64-8e87-3d72b2744d88, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.092744036Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.092793797Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 648e5f66-03f8-489e-a2bb-91cee8654a72, dataset_name: DatasetName { pool_name: ZpoolName { id: 82347231-68ab-4934-8007-b28d9a3a3777, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.110673072Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.110744503Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 525c4f1a-5d69-4ecf-a43b-162fb1134523, dataset_name: DatasetName { pool_name: ZpoolName { id: 28899eb6-914e-44b6-b77e-9d82d92186d6, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.128621757Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.128672298Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0a93b238-dc4f-4234-98ad-cf545c1b8ea6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6de8542c-123a-4abc-9a88-e72f1063a9f4, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.147190154Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.147250765Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 80d3c328-30b9-4b19-852b-d0002f908859, dataset_name: DatasetName { pool_name: ZpoolName { id: e43353e3-7e66-4e6f-bd56-19f8143ee8b5, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.165362084Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.165410435Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47a9dbe8-5005-4fee-bb98-58944f67aa76, dataset_name: DatasetName { pool_name: ZpoolName { id: 5c3019ca-328c-47ab-9982-06e092199db8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.183685937Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.183723787Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06241231-d15c-4adb-ae5b-e8e364e35528, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c92b6bd-8180-45a3-9123-a99ddc649159, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.204226139Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.204738819Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 22aed52b-11e9-4a7d-bebf-3356da024fc9, dataset_name: DatasetName { pool_name: ZpoolName { id: ea19d3d4-e058-46fb-981b-9ccb255f563c, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.220353112Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.220395333Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 57f3dc22-0a8a-40ec-a7b7-137531501134, dataset_name: DatasetName { pool_name: ZpoolName { id: e22df0ad-bbf8-4902-93d6-e67ff23c3234, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.23840168Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.238450991Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6eebe2cc-6b03-4cdd-bd4d-26af275d74bc, dataset_name: DatasetName { pool_name: ZpoolName { id: c7e0e489-6c2b-472e-8a1f-efe83bb548dd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.255806996Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.255861147Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.256148122Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.256327815Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.343302134Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.379805486Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.380213804Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.380247494Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.404022706Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.404396033Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.404429423Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.429726539Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.429775503Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.429788773Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.579804286Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.58002068Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.580056821Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.594987032Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.595205026Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.595282679Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.630565677Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.630609938Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.630614738Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.642432183Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.647656978Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.652730219Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.657478516Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.663160719Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.668040948Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.673532267Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.808772362Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.813542278Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.818526879Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.823182643Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.827820308Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.832197757Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.837219518Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Configuring new Omicron zone: oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.842222839Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.846636439Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.851239922Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.856901535Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.87589096Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.880731758Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.885517085Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.891375371Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.896739128Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.902139466Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.906918763Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.911501506Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.916312363Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.920856676Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:26.976872943Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.0152805Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.058873721Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.100274453Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.133601037Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.134882811Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.134897701Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.193721429Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 133, 194]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte1\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.193776979Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:1277","zone_type":"nexus"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.196613164Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.196629176Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.252689759Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 254, 213]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte2\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.25275084Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:1277","zone_type":"external_dns"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.254753196Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.254765957Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.309960568Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 195, 126]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte3\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.31002592Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:1277","zone_type":"external_dns"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.312127408Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.312167128Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.364150516Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 246, 195]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte4\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.364206893Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:1277","zone_type":"nexus"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.367736217Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.367749707Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.419814432Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 171, 116]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte5\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.419869293Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:1277","zone_type":"nexus"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.1.167,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.420471954Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.1.167,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.420483444Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.1.167,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.428758164Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.431484364Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.1.164,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.456091261Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.1.164,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.456167112Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.1.165,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.456369976Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.1.165,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.456387576Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.1.166,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.456665601Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.1.166,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.456681191Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.1.168,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.456887245Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.1.168,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.456895274Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.1.164,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.457022908Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.1.165,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.459120706Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.1.166,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.461181173Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.461197343Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.461208113Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.1.168,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.46431549Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.466327826Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.466333766Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.555674468Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.560583357Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.565564027Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.57119676Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.577373732Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.583596945Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.589166786Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.595289027Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.600873368Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:27.60701482Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService14\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::11\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::11\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"17000\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:30.747941466Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService13\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::10\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::10\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"17000\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:30.748183311Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:30.748229091Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService15\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::f\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::f\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"17000\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:30.756851828Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:30.757395338Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:30.786557757Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService17\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::18\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_ea19d3d4-e058-46fb-981b-9ccb255f563c/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::18\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"093e299e-c70c-49a4-b8d2-cdbc7519b070\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:30.915414276Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService23\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::17\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_3c92b6bd-8180-45a3-9123-a99ddc649159/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::17\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"444b65ae-5031-4e1b-9878-1efb6b08ce20\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:30.915601569Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:30.915664341Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService18\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::14\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_6de8542c-123a-4abc-9a88-e72f1063a9f4/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::14\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"d3ad92e9-addf-47ad-ae73-22b9743a83f5\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:30.923561784Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService20\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::12\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_82347231-68ab-4934-8007-b28d9a3a3777/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::12\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"8cd13b47-e572-4625-8022-57c0f24dfaa0\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:30.923812718Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService21\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::13\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_28899eb6-914e-44b6-b77e-9d82d92186d6/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::13\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"a9995da8-953f-4ad9-8cf8-b484e080974b\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:30.924022272Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService22\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::15\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_e43353e3-7e66-4e6f-bd56-19f8143ee8b5/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::15\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"c75657d7-3cd4-435d-981a-11201eeec233\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:30.924213056Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:30.924222306Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService10\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::19\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_e22df0ad-bbf8-4902-93d6-e67ff23c3234/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::19\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"96d123ae-b766-40aa-b115-103d9ba68b91\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:30.93162172Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService11\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::1a\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_c7e0e489-6c2b-472e-8a1f-efe83bb548dd/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::1a\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"ee6966f6-a047-4b26-be03-a11646ea767c\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:30.931806393Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService16\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::16\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_5c3019ca-328c-47ab-9982-06e092199db8/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::16\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"3f5dbb2e-6848-41fc-ba03-a3eb231fa74a\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:30.932094229Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:30.932109899Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:30.939304589Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:30.945821308Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:30.952707953Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:30.959231161Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:30.965225002Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:30.973327027Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService19\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::d\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/oximeter\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"id\" value=\"b6a17b0f-0906-4eb9-94c3-528d000f68dc\"/>\n        <propval type=\"astring\" name=\"address\" value=\"[fd00:1122:3344:101::d]:12223\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:31.326792843Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:31.327018096Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\") failed: Property not found. retry in 48.854819ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:31.603146647Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\") failed: Property not found. retry in 29.875273ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:31.622464208Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\") failed: Property not found. retry in 40.580244ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:31.666533898Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\") failed: Property not found. retry in 62.925229ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:31.679579145Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\") failed: Property not found. retry in 67.798406ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:31.692689843Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\") failed: Property not found. retry in 62.734311ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:31.812963995Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\") failed: Property not found. retry in 72.366857ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:32.086733915Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\") failed: Property not found. retry in 54.412418ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:32.252482523Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\") failed: Property not found. retry in 55.11272ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:41.397890738Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\") failed: Property not found. retry in 41.500283ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:41.795883264Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\") failed: Property not found. retry in 65.224797ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:41.821228397Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\") failed: Property not found. retry in 70.296942ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:41.837078796Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\") failed: Property not found. retry in 35.705639ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:41.85423354Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\") failed: Property not found. retry in 126.727812ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.054941489Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\") failed: Property not found. retry in 87.328966ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.117662734Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\") failed: Property not found. retry in 144.882375ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.193088199Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\") failed: Property not found. retry in 95.155341ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.220377483Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\") failed: Property not found. retry in 72.610248ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.247552105Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\") failed: Property not found. retry in 83.881003ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.278098458Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\") failed: Property not found. retry in 115.95931ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.293839872Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\") failed: Property not found. retry in 95.143748ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.311590444Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\") failed: Property not found. retry in 124.505321ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.326491986Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\") failed: Property not found. retry in 106.449786ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.340882514Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\") failed: Property not found. retry in 105.231135ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.354816446Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\") failed: Property not found. retry in 65.211443ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.370114913Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\") failed: Property not found. retry in 50.75902ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.383401093Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\") failed: Property not found. retry in 250.999964ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.397671124Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\") failed: Property not found. retry in 160.761524ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.41304347Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\") failed: Property not found. retry in 242.321793ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.427036463Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\") failed: Property not found. retry in 194.089351ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.443349848Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\") failed: Property not found. retry in 219.994138ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.457455884Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\") failed: Property not found. retry in 212.469413ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.471461257Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\") failed: Property not found. retry in 127.550512ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.496374288Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\") failed: Property not found. retry in 268.821116ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.51254977Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\") failed: Property not found. retry in 199.333164ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.528391717Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\") failed: Property not found. retry in 253.552384ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.545004308Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\") failed: Property not found. retry in 235.896515ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.559875707Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\") failed: Property not found. retry in 192.323902ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.574943679Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\") failed: Property not found. retry in 100.072973ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.592982646Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\") failed: Property not found. retry in 454.688091ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.608834633Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\") failed: Property not found. retry in 319.944679ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.656098351Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\") failed: Property not found. retry in 372.917834ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.67445264Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\") failed: Property not found. retry in 399.307918ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.687742451Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\") failed: Property not found. retry in 288.464056ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.701712281Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\") failed: Property not found. retry in 356.339504ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.715445572Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\") failed: Property not found. retry in 485.043168ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.728156652Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:42.733556556Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56239"}
{"msg":"wait for service Some(\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\") failed: Property not found. retry in 241.896835ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.744266914Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\") failed: Property not found. retry in 364.191695ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.760200432Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:42.779257967Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"0f5c119f-9e68-43c5-b9b2-a68fde39eb43","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56239","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":34239,"response_code":"200"}
{"msg":"wait for service Some(\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\") failed: Property not found. retry in 401.911969ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.789789448Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\") failed: Property not found. retry in 259.358962ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.80651104Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\") failed: Property not found. retry in 370.154221ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.820429612Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\") failed: Property not found. retry in 290.21036ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.833935617Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\") failed: Property not found. retry in 1.006306015s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:42.991686972Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\") failed: Property not found. retry in 1.039892101s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:43.010717716Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\") failed: Property not found. retry in 1.15198777s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:43.028185372Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\") failed: Property not found. retry in 812.851166ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:43.064098552Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\") failed: Property not found. retry in 1.087742599s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:43.081625909Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\") failed: Property not found. retry in 769.566436ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:43.098554226Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\") failed: Property not found. retry in 853.434585ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:43.112114651Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\") failed: Property not found. retry in 1.064721091s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:43.12640676Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\") failed: Property not found. retry in 786.543892ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:43.141584636Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\") failed: Property not found. retry in 1.120239206s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:43.154474258Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\") failed: Property not found. retry in 502.344179ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:43.208049467Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\") failed: Property not found. retry in 626.318066ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:43.220575204Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\") failed: Property not found. retry in 408.09342ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:43.232986358Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\") failed: Property not found. retry in 1.184453253s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:43.65471241Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\") failed: Property not found. retry in 692.218586ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:43.724460922Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService25\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::8\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte2\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.1.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.1.5\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/external_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:101::8]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"172.30.1.5:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:43.811284674Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService26\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::9\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte3\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.1.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.1.6\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/external_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:101::9]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"172.30.1.6:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:43.811920345Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:43.814644754Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:43.821773834Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\") failed: Property not found. retry in 860.964629ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:44.43768686Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\") failed: Property not found. retry in 1.48850656s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:44.455217777Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\") failed: Property not found. retry in 1.238240603s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:44.470944812Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\") failed: Property not found. retry in 1.214371553s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:44.489183752Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\") failed: Property not found. retry in 1.231055732s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:44.502210748Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\") failed: Property not found. retry in 1.499581011s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:44.514874047Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\") failed: Property not found. retry in 719.235483ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:44.528128017Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\") failed: Property not found. retry in 956.387277ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:44.54212316Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\") failed: Property not found. retry in 978.915256ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:44.559422573Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\") failed: Property not found. retry in 1.139395781s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:44.573398226Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\") failed: Property not found. retry in 1.084220078s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:46.251995021Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\") failed: Property not found. retry in 518.084503ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:46.299623665Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\") failed: Property not found. retry in 930.126886ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:46.338228394Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\") failed: Property not found. retry in 1.004925005s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:46.366922173Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\") failed: Property not found. retry in 1.460067721s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:46.382004026Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\") failed: Property not found. retry in 1.356575561s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:46.396626951Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\") failed: Property not found. retry in 1.340260585s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:46.411086282Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\") failed: Property not found. retry in 766.971167ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:46.423324494Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\") failed: Property not found. retry in 511.922532ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:46.435665437Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\") failed: Property not found. retry in 509.629317ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:46.455596798Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\") failed: Property not found. retry in 1.446829192s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:46.47153086Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\") failed: Property not found. retry in 722.75616ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:46.489177755Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\") failed: Property not found. retry in 651.824507ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:46.507192971Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\") failed: Property not found. retry in 56.448242ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:46.522660071Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\") failed: Property not found. retry in 34.800518ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:46.540895211Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\") failed: Property not found. retry in 108.533546ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:46.591173101Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\") failed: Property not found. retry in 102.964477ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:46.603892781Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\") failed: Property not found. retry in 152.912413ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:46.726027932Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\") failed: Property not found. retry in 217.783563ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:46.753528889Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\") failed: Property not found. retry in 1.45192266s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:46.841941569Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\") failed: Property not found. retry in 216.511509ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:46.938913644Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\") failed: Property not found. retry in 1.388223128s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:46.973151874Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\") failed: Property not found. retry in 386.307266ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:47.016651401Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\") failed: Property not found. retry in 954.083643ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:47.043262243Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\") failed: Property not found. retry in 544.180899ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:47.168259955Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\") failed: Property not found. retry in 515.880689ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:47.178729654Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\") failed: Property not found. retry in 713.473874ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:47.201885855Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\") failed: Property not found. retry in 928.793746ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:47.223500744Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\") failed: Property not found. retry in 1.251418421s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:47.279608478Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\") failed: Property not found. retry in 622.624021ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:47.348648439Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\") failed: Property not found. retry in 1.115550903s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:47.383911661Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\") failed: Property not found. retry in 498.286868ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:47.41440899Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\") failed: Property not found. retry in 842.775261ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:47.733337409Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\") failed: Property not found. retry in 856.942977ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:47.775948602Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\") failed: Property not found. retry in 898.314049ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:47.792876199Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\") failed: Property not found. retry in 1.275246825s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:47.808678114Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\") failed: Property not found. retry in 577.148565ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:47.864694948Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\") failed: Property not found. retry in 960.391527ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:47.941886465Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\") failed: Property not found. retry in 607.053831ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:47.973352015Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\") failed: Property not found. retry in 1.34827484s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:47.991175506Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\") failed: Property not found. retry in 577.839981ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:48.009699032Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\") failed: Property not found. retry in 1.193741162s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:48.035284166Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\") failed: Property not found. retry in 1.481247608s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:48.172029094Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\") failed: Property not found. retry in 501.942024ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:48.313368528Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\") failed: Property not found. retry in 1.070280999s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:48.382155583Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\") failed: Property not found. retry in 876.693067ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:48.462722714Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\") failed: Property not found. retry in 852.957789ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:48.521680258Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\") failed: Property not found. retry in 920.8774ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:48.550198164Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\") failed: Property not found. retry in 500.703269ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:48.592403917Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\") failed: Property not found. retry in 867.086497ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:48.607129374Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\") failed: Property not found. retry in 1.126690302s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:48.623977309Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\") failed: Property not found. retry in 791.073223ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:48.65220676Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService28\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::c\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte4\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.2.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.2.7\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:48.6748453Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:48.675926269Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\") failed: Property not found. retry in 742.43211ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:48.71134256Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService27\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::a\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte5\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.2.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.2.5\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:48.763368282Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService24\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::b\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte1\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.2.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.2.6\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:48.763769539Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:48.76386308Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:48.772637379Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\") failed: Property not found. retry in 648.956655ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:48.832807038Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\") failed: Property not found. retry in 525.717326ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.030840272Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\") failed: Property not found. retry in 1.109523416s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.122128284Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\") failed: Property not found. retry in 1.133010599s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.170688553Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\") failed: Property not found. retry in 926.191246ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.255987317Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\") failed: Property not found. retry in 72.051078ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.302689732Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\") failed: Property not found. retry in 1.009743559s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.356767333Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\") failed: Property not found. retry in 1.195437473s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.372181889Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\") failed: Property not found. retry in 31.059989ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.388641126Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\") failed: Property not found. retry in 40.364055ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.40259701Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\") failed: Property not found. retry in 929.285309ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.421757137Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\") failed: Property not found. retry in 74.06991ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.434367505Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\") failed: Property not found. retry in 87.433779ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.447606725Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\") failed: Property not found. retry in 864.665969ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.460227383Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\") failed: Property not found. retry in 86.519606ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.474317248Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\") failed: Property not found. retry in 1.357652308s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.489053924Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\") failed: Property not found. retry in 625.491586ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.501803125Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\") failed: Property not found. retry in 1.088387289s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.516124154Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\") failed: Property not found. retry in 810.693438ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.529106959Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\") failed: Property not found. retry in 284.394451ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.540546587Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\") failed: Property not found. retry in 758.48917ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.553223166Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\") failed: Property not found. retry in 113.861558ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.565919045Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\") failed: Property not found. retry in 735.545721ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.580391327Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\") failed: Property not found. retry in 206.751449ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.594293768Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\") failed: Property not found. retry in 786.794678ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.670301194Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\") failed: Property not found. retry in 559.924127ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.708672648Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\") failed: Property not found. retry in 933.752032ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.766536945Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\") failed: Property not found. retry in 407.740277ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.817968096Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\") failed: Property not found. retry in 240.675734ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:49.840615376Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\") failed: Property not found. retry in 674.712419ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:50.095901496Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\") failed: Property not found. retry in 1.193386916s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:50.139412773Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\") failed: Property not found. retry in 1.342913954s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:50.194816436Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\") failed: Property not found. retry in 621.319614ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:50.242577379Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\") failed: Property not found. retry in 522.068889ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:50.257077523Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\") failed: Property not found. retry in 826.340998ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:50.283704995Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\") failed: Property not found. retry in 1.110576301s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:50.317394894Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\") failed: Property not found. retry in 590.321521ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:50.330740436Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\") failed: Property not found. retry in 892.744747ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:50.34480789Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\") failed: Property not found. retry in 941.866176ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:50.364265743Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\") failed: Property not found. retry in 958.934006ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:50.379452608Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\") failed: Property not found. retry in 1.395068113s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:50.395195492Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\") failed: Property not found. retry in 1.122878226s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:50.407539506Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService12\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::e\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/clickhouse\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::e\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"8123\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:50.411940295Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:50.412501206Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\") failed: Property not found. retry in 873.205795ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:50.470949153Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\") failed: Property not found. retry in 1.383492687s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:50.584409189Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\") failed: Property not found. retry in 1.318778892s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:50.619222796Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\") failed: Property not found. retry in 49.756494ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:50.712576166Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\") failed: Property not found. retry in 663.52308ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:50.72553066Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\") failed: Property not found. retry in 67.084419ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:50.77412913Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\") failed: Property not found. retry in 1.350247651s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:50.789836504Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\") failed: Property not found. retry in 552.743464ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:50.804455933Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\") failed: Property not found. retry in 281.738688ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:50.889538358Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\") failed: Property not found. retry in 598.703891ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:50.961243196Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\") failed: Property not found. retry in 1.054460542s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:50.997470392Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\") failed: Property not found. retry in 1.001199931s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:51.010028119Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\") failed: Property not found. retry in 647.878086ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:51.125737373Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\") failed: Property not found. retry in 561.959446ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:51.189465206Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\") failed: Property not found. retry in 1.259055041s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:51.254237278Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\") failed: Property not found. retry in 757.126852ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:51.321336173Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\") failed: Property not found. retry in 1.14937169s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:51.348064742Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\") failed: Property not found. retry in 1.225128082s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:51.363247301Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\") failed: Property not found. retry in 1.069677643s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:51.377686942Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\") failed: Property not found. retry in 1.244455621s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:51.391643675Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\") failed: Property not found. retry in 928.461155ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:51.405218441Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\") failed: Property not found. retry in 1.212850813s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:51.445280866Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\") failed: Property not found. retry in 846.90682ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:51.554936765Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\") failed: Property not found. retry in 1.265656003s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:51.572055619Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\") failed: Property not found. retry in 531.887776ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:51.587948319Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\") failed: Property not found. retry in 465.163202ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:51.768282342Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\") failed: Property not found. retry in 1.368238215s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:51.79037463Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\") failed: Property not found. retry in 957.667099ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:51.809355684Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\") failed: Property not found. retry in 753.571852ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:51.954721774Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\") failed: Property not found. retry in 1.4630397s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:51.985620594Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\") failed: Property not found. retry in 1.495544305s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:52.030644558Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\") failed: Property not found. retry in 879.913956ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:52.06997419Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\") failed: Property not found. retry in 513.943085ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:52.09315178Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\") failed: Property not found. retry in 937.768ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:52.137577413Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\") failed: Property not found. retry in 890.475846ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:52.158661525Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\") failed: Property not found. retry in 1.148184701s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:52.250729221Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\") failed: Property not found. retry in 1.330642865s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:52.347534023Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\") failed: Property not found. retry in 1.2952067s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:52.418730852Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\") failed: Property not found. retry in 1.111690096s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:52.462592126Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\") failed: Property not found. retry in 673.489828ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:52.512441697Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\") failed: Property not found. retry in 779.244837ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:52.528679531Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\") failed: Property not found. retry in 1.261988166s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:52.60155231Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\") failed: Property not found. retry in 1.15123934s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:52.621964097Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\") failed: Property not found. retry in 600.841797ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:52.652613204Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\") failed: Property not found. retry in 1.469196731s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:52.672037236Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\") failed: Property not found. retry in 1.488220743s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:52.727840475Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\") failed: Property not found. retry in 914.508387ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:52.781763691Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\") failed: Property not found. retry in 1.188288806s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:52.853679833Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\") failed: Property not found. retry in 1.018185023s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:52.96678659Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\") failed: Property not found. retry in 1.491739243s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:53.063477169Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\") failed: Property not found. retry in 631.053681ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:53.090290975Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\") failed: Property not found. retry in 975.001072ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:53.172725557Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76\") failed: Property not found. retry in 1.398144517s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:53.201941605Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5\") failed: Property not found. retry in 975.028002ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:53.268977119Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\") failed: Property not found. retry in 1.439119584s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:53.323687189Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\") failed: Property not found. retry in 746.398795ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:53.420426489Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6\") failed: Property not found. retry in 1.164890165s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:53.464807772Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\") failed: Property not found. retry in 1.101945469s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:53.543281243Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc\") failed: Property not found. retry in 1.018729259s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:53.590459806Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523\") failed: Property not found. retry in 1.452519585s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:53.701080498Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134\") failed: Property not found. retry in 1.407177838s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:53.719504332Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc\") failed: Property not found. retry in 1.429982148s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:53.734493983Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82\") failed: Property not found. retry in 928.333826ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:53.750666386Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\") failed: Property not found. retry in 586.17688ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:53.788013942Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce\") failed: Property not found. retry in 1.015024224s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:53.883101655Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\") failed: Property not found. retry in 709.687754ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:54.006812571Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9\") failed: Property not found. retry in 1.21256322s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:54.057542699Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72\") failed: Property not found. retry in 979.147789ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:54.163897324Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\") failed: Property not found. retry in 858.598696ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:54.186964575Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\") failed: Property not found. retry in 629.816847ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:54.213893929Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528\") failed: Property not found. retry in 608.188628ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:54.240428779Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\") failed: Property not found. retry in 1.255936753s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:54.401998043Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\") failed: Property not found. retry in 642.168051ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:54.576518901Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\") failed: Property not found. retry in 831.394635ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:54.710656418Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\") failed: Property not found. retry in 1.304996291s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:54.763507059Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859\") failed: Property not found. retry in 1.276663338s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:54.785007914Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\") failed: Property not found. retry in 929.042317ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:54.863616706Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\") failed: Property not found. retry in 685.095962ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:55.073256499Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\") failed: Property not found. retry in 1.43124304s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:55.452460623Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\") failed: Property not found. retry in 980.084573ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:55.575224264Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\") failed: Property not found. retry in 901.831747ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:55.685890277Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\") failed: Property not found. retry in 1.041177085s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:55.785728544Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\") failed: Property not found. retry in 1.346344462s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:55.813807572Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\") failed: Property not found. retry in 658.92591ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:56.108148702Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f\") failed: Property not found. retry in 1.411274044s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:56.577627115Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\") failed: Property not found. retry in 1.043427088s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:56.606273825Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\") failed: Property not found. retry in 1.148763911s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:56.781294751Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\") failed: Property not found. retry in 641.737632ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:56.841044612Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\") failed: Property not found. retry in 1.273473164s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:56.897920817Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\") failed: Property not found. retry in 1.011669798s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:57.176773078Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\") failed: Property not found. retry in 637.038104ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:57.497705026Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\") failed: Property not found. retry in 843.893913ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:57.943188058Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\") failed: Property not found. retry in 912.540349ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:58.148022715Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\") failed: Property not found. retry in 1.191569795s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:58.183290535Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\") failed: Property not found. retry in 693.127848ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:58.199296612Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88\") failed: Property not found. retry in 792.693202ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:58.903775161Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:44:59.318542398Z","hostname":"buskin","pid":1618,"component":"NexusNotifierTask","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.721555411Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_nexus_3b85f16b-9ecd-4867-ae56-7f8a1f492a4d"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.721793805Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_crucible_57f3dc22-0a8a-40ec-a7b7-137531501134"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.721810296Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_crucible_6eebe2cc-6b03-4cdd-bd4d-26af275d74bc"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.721826256Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_external_dns_57425cdc-844b-42e3-b45f-c8fb7871161f"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.721834106Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_clickhouse_8bf3945e-e049-4e64-8e87-3d72b2744d88"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.721841416Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_crucible_pantry_23c9f9de-3736-4878-ba3b-118cebaacb82"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.721849046Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_external_dns_dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.721855897Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_crucible_pantry_a684a100-6031-4b97-8de3-41e15037fbd5"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.72204245Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_crucible_pantry_f4d5df0a-748d-40ab-8235-7c927aae50ce"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.72205535Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_crucible_47a9dbe8-5005-4fee-bb98-58944f67aa76"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.7220676Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_crucible_22aed52b-11e9-4a7d-bebf-3356da024fc9"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.722075601Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_crucible_0a93b238-dc4f-4234-98ad-cf545c1b8ea6"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.722082991Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_nexus_f33d72e6-7ea4-4e54-8bfc-dc4233c612c4"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.722090531Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_oximeter_b6a17b0f-0906-4eb9-94c3-528d000f68dc"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.722113081Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_crucible_648e5f66-03f8-489e-a2bb-91cee8654a72"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.722120921Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_nexus_cca30d41-f76f-4c5a-92f4-bb1cde1c17bd"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.722128451Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_crucible_525c4f1a-5d69-4ecf-a43b-162fb1134523"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.722135762Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_crucible_80d3c328-30b9-4b19-852b-d0002f908859"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.722142712Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_crucible_06241231-d15c-4adb-ae5b-e8e364e35528"}
{"msg":"Writing ledger to /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.72645497Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.729636726Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.736677535Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"3e82e629-fb4e-4785-8c27-47194697b0c4","remote_addr":"[fd00:1122:3344:101::1]:52853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":33893379,"response_code":"204"}
{"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.765032528Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1230"}
{"msg":"Writing ledger to /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.765253292Z","hostname":"buskin","pid":1618,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.765424185Z","hostname":"buskin","pid":1618,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.768648807Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:677"}
{"msg":"Nexus address: [fd00:1122:3344:101::b]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.772820589Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:685"}
{"msg":"rack_network_config: RackNetworkConfigV1 {\n    bfd: [],\n    bgp: [],\n    infra_ip_first: 10.151.1.174,\n    infra_ip_last: 10.151.1.174,\n    ports: [\n        PortConfigV1 {\n            addresses: [\n                V4(\n                    Ipv4Network {\n                        addr: 10.151.1.174,\n                        prefix: 24,\n                    },\n                ),\n            ],\n            autoneg: false,\n            bgp_peers: [],\n            port: \"qsfp0\",\n            routes: [\n                RouteConfig {\n                    destination: V4(\n                        Ipv4Network {\n                            addr: 0.0.0.0,\n                            prefix: 0,\n                        },\n                    ),\n                    nexthop: 10.151.1.100,\n                },\n            ],\n            switch: Switch0,\n            uplink_port_fec: None,\n            uplink_port_speed: Speed40G,\n        },\n    ],\n    rack_subnet: Ipv6Network {\n        addr: fd00:1122:3344:100::,\n        prefix: 56,\n    },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.818713559Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:816"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/98531190-dbea-48cf-b76b-c50c819faca1/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.828969075Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/98531190-dbea-48cf-b76b-c50c819faca1/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:44:59.949315313Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/98531190-dbea-48cf-b76b-c50c819faca1/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:00.017723911Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/98531190-dbea-48cf-b76b-c50c819faca1/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:00.191419294Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/98531190-dbea-48cf-b76b-c50c819faca1/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:00.366963851Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:45:00.451376509Z","hostname":"buskin","pid":1618,"component":"NexusNotifierTask","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/98531190-dbea-48cf-b76b-c50c819faca1/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:00.587811988Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/98531190-dbea-48cf-b76b-c50c819faca1/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:00.858280213Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/98531190-dbea-48cf-b76b-c50c819faca1/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:01.067785337Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/98531190-dbea-48cf-b76b-c50c819faca1/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:01.32339543Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/98531190-dbea-48cf-b76b-c50c819faca1/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:01.817164936Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:45:02.445636559Z","hostname":"buskin","pid":1618,"component":"NexusNotifierTask","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/98531190-dbea-48cf-b76b-c50c819faca1/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:02.542407541Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/98531190-dbea-48cf-b76b-c50c819faca1/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:03.403939554Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/98531190-dbea-48cf-b76b-c50c819faca1/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:03.934324333Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:45:04.444725052Z","hostname":"buskin","pid":1618,"component":"NexusNotifierTask","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/98531190-dbea-48cf-b76b-c50c819faca1/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:04.764370727Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/98531190-dbea-48cf-b76b-c50c819faca1/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:05.359805805Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/98531190-dbea-48cf-b76b-c50c819faca1/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:06.240294543Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:45:06.442779838Z","hostname":"buskin","pid":1618,"component":"NexusNotifierTask","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/93b0cc91-dc62-481c-9ea1-425d29f058ae): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:45:08.443141386Z","hostname":"buskin","pid":1618,"component":"NexusNotifierTask","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/98531190-dbea-48cf-b76b-c50c819faca1/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:08.455396108Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:45:10.46376629Z","hostname":"buskin","pid":1618,"sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1178","retry_after":"20.061729981s","error":"\"Metric registration error: Error Response: status: 503 Service Unavailable; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"e002f9fd-989d-40a4-b33b-a8415e04cb2a\\\", \\\"content-length\\\": \\\"133\\\", \\\"date\\\": \\\"Sat, 30 Mar 2024 05:45:09 GMT\\\"}; value: Error { error_code: Some(\\\"ServiceNotAvailable\\\"), message: \\\"Service Unavailable\\\", request_id: \\\"e002f9fd-989d-40a4-b33b-a8415e04cb2a\\\" }\""}
{"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"51b41c50-860b-4403-b011-e7f74ead199f\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 05:45:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"51b41c50-860b-4403-b011-e7f74ead199f\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:11.336239612Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:12.492344679Z","hostname":"buskin","pid":1618,"component":"NexusNotifierTask","sled_id":"93b0cc91-dc62-481c-9ea1-425d29f058ae","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:12.832123088Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36942"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:12.834582762Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"d8edecdf-eda9-4298-9029-61fb32ccbafd","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36942","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1361,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:13.503408519Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:61863"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:13.504475338Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"ac0e60c8-4224-49aa-b5c3-54a0d4a26778","remote_addr":"[fd00:1122:3344:101::b]:61863","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1039,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:14.459029336Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:34422"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:14.465983022Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 72eae3cb-c4dd-4f77-8b5b-946c1c2c7c41, kind: Service { id: dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8 }, name: Name(\"external-dns-dbd59cbf-fa3e-4fe2-b758-b2a136ae25a8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9701e6c2-1d46-4a7d-b6e3-8b3125f74122, kind: Service { id: 57425cdc-844b-42e3-b45f-c8fb7871161f }, name: Name(\"external-dns-57425cdc-844b-42e3-b45f-c8fb7871161f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 213])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 21dc6cd8-6a7b-4ff0-9dd6-a2e7dc0c2a56, kind: Service { id: cca30d41-f76f-4c5a-92f4-bb1cde1c17bd }, name: Name(\"nexus-cca30d41-f76f-4c5a-92f4-bb1cde1c17bd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 195])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 3630ad58-6099-41a3-8374-fe9c065b9ec4, kind: Service { id: f33d72e6-7ea4-4e54-8bfc-dc4233c612c4 }, name: Name(\"nexus-f33d72e6-7ea4-4e54-8bfc-dc4233c612c4\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 591e0b7a-a507-4ecd-a23f-18a94999cc3e, kind: Service { id: 3b85f16b-9ecd-4867-ae56-7f8a1f492a4d }, name: Name(\"nexus-3b85f16b-9ecd-4867-ae56-7f8a1f492a4d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 194])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:14.466468391Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:14.466724725Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:14.466735996Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:14.466749306Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:14.466768986Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:14.466781336Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:14.466810927Z","hostname":"buskin","pid":1618,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"89cf9bbc-87f0-4d35-ad08-47af3e741db3","remote_addr":"[fd00:1122:3344:101::b]:34422","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7557,"response_code":"204"}
{"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:14.468882864Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:883"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:14.691788978Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:37621"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:14.69298341Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:14.781321269Z","hostname":"buskin","pid":1618,"uri":"/inventory","method":"GET","req_id":"9ec194f5-0819-4af8-a70a-874a9daaee54","remote_addr":"[fd00:1122:3344:101::b]:37621","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":89474,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:14.782027871Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:14.782121553Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:14.78360627Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:14.7852769Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"GET","req_id":"10c19d31-6c6e-4bd6-bf11-718523716989","remote_addr":"[fd00:1122:3344:101::b]:37621","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3708,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:38.974058275Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:40939"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:39.080019313Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"9dbe9981-afd9-422f-a065-530a750b1f76","remote_addr":"[fd00:1122:3344:101::b]:40939","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":105889,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:39.080292308Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:46592"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:39.080469381Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"00264ac6-4547-4bc8-884d-08ed3e0f46c5","remote_addr":"[fd00:1122:3344:101::b]:40939","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":212,"response_code":"200"}
{"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:39.109892552Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"}
{"msg":"Writing ledger to /pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:39.11007594Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:39.11188345Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:39.112293407Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:39.112315958Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"PUT","req_id":"b2cee4fe-7a4e-458e-a323-68f730a1f0d3","remote_addr":"[fd00:1122:3344:101::b]:40939","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3786,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:39.139507399Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:59216"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:39.142099537Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"c3bee200-cc6b-4dee-a04c-2c434e149428","remote_addr":"[fd00:1122:3344:101::a]:46592","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":61652,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:39.146109449Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"5cf4cddb-53f2-49ca-b924-37fa60f2a4be","remote_addr":"[fd00:1122:3344:101::a]:46592","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":636,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:39.19256658Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"d7d52828-68e5-40ed-ba8f-7c5b1e6ccb11","remote_addr":"[fd00:1122:3344:101::c]:59216","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":52927,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:39.192939297Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"c959bfde-5c7b-43f7-b7b8-7c18236970d1","remote_addr":"[fd00:1122:3344:101::c]:59216","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":144,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:42.870431952Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63431"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:45:42.870538854Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"f6363e42-d012-4efc-b3dc-12b97d1b47fc","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63431","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":70,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:00.602194384Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::d]:56676"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:00.611597194Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/93b0cc91-dc62-481c-9ea1-425d29f058ae","method":"GET","req_id":"e58bbf94-06b1-46f9-a6a5-6da03eb3e6fa","remote_addr":"[fd00:1122:3344:101::d]:56676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9353,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:09.141663247Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:51346"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:09.192761582Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"b8d11dc1-1583-4682-b0e1-c4b900a4278a","remote_addr":"[fd00:1122:3344:101::b]:51346","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":51022,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:09.193950753Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"73da25f5-4d9b-42d4-bbb2-97b0fac04fdc","remote_addr":"[fd00:1122:3344:101::b]:51346","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":977,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:09.260666707Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:57762"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:09.306743755Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"6556da29-14e6-4302-9e92-1644a9d47d50","remote_addr":"[fd00:1122:3344:101::a]:57762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45922,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:09.307004739Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"aedf4b1e-41dd-40f2-ac16-ed07d1b42948","remote_addr":"[fd00:1122:3344:101::a]:57762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":104,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:09.483048285Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:62543"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:09.528646601Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"14b1c8ee-150e-4ea3-b62a-65c104f2eede","remote_addr":"[fd00:1122:3344:101::c]:62543","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45508,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:09.529564557Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"8d21d3f3-d90f-4262-9056-041b5c9be6c6","remote_addr":"[fd00:1122:3344:101::c]:62543","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":757,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:12.912217175Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42016"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:12.912311717Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"426d25e5-b5fc-4d72-8b3e-9612aecf29ac","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42016","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":88,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:30.607691019Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/93b0cc91-dc62-481c-9ea1-425d29f058ae","method":"GET","req_id":"79942226-4c49-4276-acbd-b2112dfe98e0","remote_addr":"[fd00:1122:3344:101::d]:56676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1260,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:38.820637176Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:39353"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:38.869837046Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"7420616d-037c-4083-aa46-16995ceb81a1","remote_addr":"[fd00:1122:3344:101::a]:39353","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49130,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:38.870783673Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"3a5dbfc8-31ca-4c8d-b3e9-380129df543c","remote_addr":"[fd00:1122:3344:101::a]:39353","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":797,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:38.922223674Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:63412"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:38.966577607Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"9a1c674a-fc82-4abc-9b68-cf7fcd7804cb","remote_addr":"[fd00:1122:3344:101::c]:63412","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44265,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:38.966673219Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:54098"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:39.009936572Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"353e37d6-261e-4b51-905f-789b54871459","remote_addr":"[fd00:1122:3344:101::b]:54098","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43183,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:39.010127565Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"2c901df2-520b-4ef7-937e-6537d0f71b16","remote_addr":"[fd00:1122:3344:101::c]:63412","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":99,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:39.01036519Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"ee6bf5af-3611-41f9-b0a5-ae83e6222b39","remote_addr":"[fd00:1122:3344:101::b]:54098","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":141,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:42.949610288Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48500"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:46:42.94972465Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"d4fd3a5e-67d9-4513-a709-3adeddc79da9","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48500","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":73,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:47:00.610432301Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/93b0cc91-dc62-481c-9ea1-425d29f058ae","method":"GET","req_id":"511f9ba0-1776-4eea-9c3a-8d4623837f0e","remote_addr":"[fd00:1122:3344:101::d]:56676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":570,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:47:09.070365061Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:54067"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:47:09.116545447Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"d6a934d9-b6e6-43ad-9990-0bbe133822bb","remote_addr":"[fd00:1122:3344:101::b]:54067","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46125,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:47:09.116823762Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"fb596565-635d-4f53-aa49-8bc79822b04e","remote_addr":"[fd00:1122:3344:101::b]:54067","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":94,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:47:09.126195251Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:60917"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:47:09.171542742Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"f59a5ef5-7763-4a20-b794-a09fd44b2960","remote_addr":"[fd00:1122:3344:101::a]:60917","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45240,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:47:09.171811497Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"f61bb8bc-05cf-4f98-9974-9f9bbfc0d318","remote_addr":"[fd00:1122:3344:101::a]:60917","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":115,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:47:09.183639961Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:59631"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:47:09.227123318Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"aef18d9b-b63b-4fa9-8334-2533cfec3218","remote_addr":"[fd00:1122:3344:101::c]:59631","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43387,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:47:09.227369402Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"9c894446-905a-474e-a5eb-ece07cb58e9d","remote_addr":"[fd00:1122:3344:101::c]:59631","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":84,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:47:12.985976001Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58412"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:47:12.986204495Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"df1a53d9-e0bf-4bd1-9ef3-2409ba4e94c4","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58412","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:47:30.610161152Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/93b0cc91-dc62-481c-9ea1-425d29f058ae","method":"GET","req_id":"080bbd47-cf62-4cce-b84f-673076edcc27","remote_addr":"[fd00:1122:3344:101::d]:56676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":560,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:47:38.921711877Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:50481"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:47:38.970039302Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"88174522-8692-4834-994b-01ce015e34d4","remote_addr":"[fd00:1122:3344:101::b]:50481","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48260,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:47:38.970329487Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"4bbb829e-b583-4f66-ae47-1200a7a8bb8f","remote_addr":"[fd00:1122:3344:101::b]:50481","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":95,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:47:39.122219786Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:52498"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:47:39.168909401Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"47b4247e-4bfe-4633-b414-3554402fc309","remote_addr":"[fd00:1122:3344:101::c]:52498","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46586,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:47:39.169731606Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"14040480-69e3-4fdc-8729-98f1737e8288","remote_addr":"[fd00:1122:3344:101::c]:52498","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":657,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:47:39.184284979Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:42287"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:47:39.23181403Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"2375fd14-54ac-4e47-8432-73509000ae90","remote_addr":"[fd00:1122:3344:101::a]:42287","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47429,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:47:39.232205907Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"53313441-9d91-4e64-ab42-45cc3f65f154","remote_addr":"[fd00:1122:3344:101::a]:42287","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":137,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:47:43.02136119Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42972"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:47:43.021516602Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"6e117d56-928f-443d-bdb6-ab5e23bbfb0b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42972","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":114,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:00.610163579Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/93b0cc91-dc62-481c-9ea1-425d29f058ae","method":"GET","req_id":"88bb69ce-d103-45d8-9f4b-0c356fd8ccdb","remote_addr":"[fd00:1122:3344:101::d]:56676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":553,"response_code":"200"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:05.967018096Z","hostname":"buskin","pid":1618,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:09.185480958Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:49048"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:09.23252333Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"33307c6b-3ebb-4a2f-81d7-147eedd85e2a","remote_addr":"[fd00:1122:3344:101::c]:49048","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46912,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:09.232821995Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"b40f0bad-60ea-4170-bffd-e27165e6307e","remote_addr":"[fd00:1122:3344:101::c]:49048","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":112,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:09.242528411Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:64697"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:09.285825045Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"5111306f-3fdf-40ba-8d85-49c3317a4801","remote_addr":"[fd00:1122:3344:101::b]:64697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43207,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:09.286061869Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"38e04c74-276f-4629-ad12-f83b7e819061","remote_addr":"[fd00:1122:3344:101::b]:64697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":75,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:09.854574639Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:55364"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:09.902892163Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"ed42fa22-2e1f-41ea-bbd7-4fa536260f6b","remote_addr":"[fd00:1122:3344:101::a]:55364","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48147,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:09.903152208Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"db2160be-9277-44cf-9df5-53f540a2c948","remote_addr":"[fd00:1122:3344:101::a]:55364","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":102,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:13.053320024Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54882"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:13.053927806Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"73cdf5e0-7947-44dc-a047-b98537053f19","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54882","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":62,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:30.611289317Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/93b0cc91-dc62-481c-9ea1-425d29f058ae","method":"GET","req_id":"fef2e675-e0c7-47e8-8a99-54038e0505db","remote_addr":"[fd00:1122:3344:101::d]:56676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":504,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:38.927005749Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:58537"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:38.975241752Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"e1bf3a36-f417-4099-b24f-a60b7ae0a4ff","remote_addr":"[fd00:1122:3344:101::c]:58537","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48165,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:38.975357914Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:61628"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:38.97901928Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"879d549c-6119-4606-b77d-ad79466437c1","remote_addr":"[fd00:1122:3344:101::c]:58537","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3437,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:39.021627872Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:46638"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:39.023609668Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"5437e840-a4c1-4897-a92c-6ed4abcca6d0","remote_addr":"[fd00:1122:3344:101::b]:61628","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48146,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:39.027982036Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"f0e51c58-5096-4eab-bbd4-ff1521dd2cc5","remote_addr":"[fd00:1122:3344:101::b]:61628","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":569,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:39.073275776Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"90c807e8-a06b-49e1-81aa-3bf199a30a55","remote_addr":"[fd00:1122:3344:101::a]:46638","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":51515,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:39.073658983Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"0603d7d0-b28a-41cd-903d-b387d8fdb061","remote_addr":"[fd00:1122:3344:101::a]:46638","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":162,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:43.086952562Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44483"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:48:43.087251868Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"46fa546e-b481-41a5-bab4-429e793f7d45","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44483","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":64,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:49:00.610692026Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/93b0cc91-dc62-481c-9ea1-425d29f058ae","method":"GET","req_id":"0b139f4f-c337-4465-903a-3d37d4390557","remote_addr":"[fd00:1122:3344:101::d]:56676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":505,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:49:09.014273789Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:57901"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:49:09.059971296Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"c4baaaed-ff4d-4eee-8323-be286ccc1430","remote_addr":"[fd00:1122:3344:101::b]:57901","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45630,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:49:09.060241461Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"fbe3d5ff-e1ef-4810-8181-be8943d5027e","remote_addr":"[fd00:1122:3344:101::b]:57901","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":101,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:49:09.172762637Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:50138"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:49:09.219751728Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"6e4ebabe-ff03-444d-b393-e739b45967ce","remote_addr":"[fd00:1122:3344:101::c]:50138","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46904,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:49:09.220022723Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"32b20fa5-4e70-4ba3-ba14-0d6f96951bc4","remote_addr":"[fd00:1122:3344:101::c]:50138","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":95,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:49:09.488285248Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:45132"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:49:09.53312194Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"d354af8e-26d8-499d-991c-337536600860","remote_addr":"[fd00:1122:3344:101::a]:45132","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44788,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:49:09.533362424Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"a9376f83-21aa-4c39-bdd2-14749da210e9","remote_addr":"[fd00:1122:3344:101::a]:45132","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":102,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:49:13.121475308Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55306"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:49:13.121566899Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"567e3145-d679-445f-a51f-9aa9894fd09f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55306","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":80,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:49:30.610743157Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/93b0cc91-dc62-481c-9ea1-425d29f058ae","method":"GET","req_id":"7dbddfbb-4054-4b9b-ae7a-f18c34c75c6b","remote_addr":"[fd00:1122:3344:101::d]:56676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":496,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:49:38.839568866Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:33001"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:49:38.887016735Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"fd9161b0-ac26-4b75-826e-1e0181ead097","remote_addr":"[fd00:1122:3344:101::b]:33001","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47389,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:49:38.88730317Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"803c0665-3bfa-4a0c-811b-b428947c844a","remote_addr":"[fd00:1122:3344:101::b]:33001","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":118,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:49:38.946232476Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:37035"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:49:38.987777048Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"20238f55-b44d-4e86-9c54-a81783980e3a","remote_addr":"[fd00:1122:3344:101::c]:37035","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":41445,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:49:38.988537542Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"8cf671c5-927b-49be-ade1-8b94b2b6f631","remote_addr":"[fd00:1122:3344:101::c]:37035","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":586,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:49:39.04367008Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:59550"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:49:39.087994133Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"9ead0c95-774d-4628-be1d-a07c212a44b6","remote_addr":"[fd00:1122:3344:101::a]:59550","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44267,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:49:39.088238447Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"6c6b0503-92f1-4143-abc0-c8ef1442ec76","remote_addr":"[fd00:1122:3344:101::a]:59550","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":99,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:49:43.153993736Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40701"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:49:43.154082857Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"aeea4a0f-0183-405a-a16c-194dee72a26c","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40701","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":90,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:50:00.611055265Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/93b0cc91-dc62-481c-9ea1-425d29f058ae","method":"GET","req_id":"44a85251-d6a0-48c9-bf3b-80e686914e53","remote_addr":"[fd00:1122:3344:101::d]:56676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":475,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:50:09.205299218Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:33829"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:50:09.248025481Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"e86e281a-d5ad-4a39-a980-7373e6268436","remote_addr":"[fd00:1122:3344:101::b]:33829","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":42663,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:50:09.248216595Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:43007"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:50:09.251737689Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"9b792743-32b5-4a44-98c8-39ae6d3b4de9","remote_addr":"[fd00:1122:3344:101::b]:33829","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3395,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:50:09.289217527Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"f415c1bc-a38b-4195-ad05-b8b1c6ccc3d3","remote_addr":"[fd00:1122:3344:101::c]:43007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":40916,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:50:09.290102303Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"33aed7cd-cb57-4d01-a75d-3f03f84ff8da","remote_addr":"[fd00:1122:3344:101::c]:43007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":689,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:50:09.299500023Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:36866"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:50:09.341262849Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"e817e582-9010-4fa9-86d2-3a80a476d05c","remote_addr":"[fd00:1122:3344:101::a]:36866","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":41681,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:50:09.341552424Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"6a902c27-989a-4f7d-915c-089b9bfe834c","remote_addr":"[fd00:1122:3344:101::a]:36866","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":99,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:50:13.186294592Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:43719"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:50:13.186462695Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"e9b85c2b-35db-4d07-9e10-2ac0c08b4e2b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:43719","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":136,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:50:30.612616283Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/93b0cc91-dc62-481c-9ea1-425d29f058ae","method":"GET","req_id":"7ee07537-df6d-4398-aa32-0a7d36df33cf","remote_addr":"[fd00:1122:3344:101::d]:56676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1289,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:50:38.917973699Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:64545"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:50:38.963568695Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"6a16d928-9845-4001-9f45-7887355cab3c","remote_addr":"[fd00:1122:3344:101::c]:64545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45543,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:50:38.963701717Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:57860"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:50:38.967571187Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"4bfa7251-4852-4ae8-b7aa-ca4f9775cbe8","remote_addr":"[fd00:1122:3344:101::c]:64545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3694,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:50:39.00970744Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"f5cbbaf7-c3a9-40db-b59b-9aefc5db787b","remote_addr":"[fd00:1122:3344:101::b]:57860","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45925,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:50:39.010057056Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"1c8e80f7-32b4-4134-8fe3-2b1a6655febb","remote_addr":"[fd00:1122:3344:101::b]:57860","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":157,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:50:39.039716423Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:51031"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:50:39.084638966Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"c6c27a70-3226-4be1-b9f2-b399cbb99d68","remote_addr":"[fd00:1122:3344:101::a]:51031","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44869,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:50:39.08487806Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"86a159a1-d3c6-4adb-95aa-d5b1ae53b435","remote_addr":"[fd00:1122:3344:101::a]:51031","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":92,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:50:43.219551608Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57819"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:50:43.21964747Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"4469fd9b-e5d4-475b-9609-fc71e130e384","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57819","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":78,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:00.611573406Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/93b0cc91-dc62-481c-9ea1-425d29f058ae","method":"GET","req_id":"2b9e90dd-3055-465e-8505-039a81eeb344","remote_addr":"[fd00:1122:3344:101::d]:56676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":490,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:09.161495976Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:36512"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:09.20643875Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"6023ecda-ca29-4b69-869f-73bb22816f16","remote_addr":"[fd00:1122:3344:101::c]:36512","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44880,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:09.207169123Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"4ea1f4f7-c772-4f44-9a7e-3d62a463fc65","remote_addr":"[fd00:1122:3344:101::c]:36512","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":592,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:09.255816963Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:48395"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:09.301776345Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"aeedcda3-9a72-4eda-a150-667e74209ffd","remote_addr":"[fd00:1122:3344:101::b]:48395","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45860,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:09.301885797Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:41833"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:09.347507633Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"1d861eda-db97-42f0-b06c-215a53f687d7","remote_addr":"[fd00:1122:3344:101::a]:41833","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45553,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:09.347712596Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"ba6f360a-47d5-434c-bbb2-6cee2fc595e2","remote_addr":"[fd00:1122:3344:101::b]:48395","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":100,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:09.34793382Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"c30266d4-cfce-47f7-809d-015efe8b30b0","remote_addr":"[fd00:1122:3344:101::a]:41833","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":139,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:13.251669277Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46485"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:13.251758128Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"d32d8be6-a0b9-4c3c-81ef-ba39093dd6df","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46485","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":72,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:30.61193784Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/93b0cc91-dc62-481c-9ea1-425d29f058ae","method":"GET","req_id":"7d0b408a-2797-45e5-b630-345ac029d97d","remote_addr":"[fd00:1122:3344:101::d]:56676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":535,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:38.828420255Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:41652"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:38.875587078Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"8fac975d-dd4f-49f1-8dc1-02516313f915","remote_addr":"[fd00:1122:3344:101::a]:41652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47100,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:38.875714881Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:45018"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:38.879745144Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"5655250e-5aec-425f-8b2f-dc4f46dd1e7c","remote_addr":"[fd00:1122:3344:101::a]:41652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3796,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:38.92707006Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"271dde93-9ea5-4527-98fb-f109c7baac98","remote_addr":"[fd00:1122:3344:101::b]:45018","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":51260,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:38.927922536Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"cea769d0-72ef-4c4a-b237-77b6bfb47058","remote_addr":"[fd00:1122:3344:101::b]:45018","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":623,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:38.936341188Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:63779"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:38.98065873Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"ceec1e54-8c97-4822-8363-06d9edeeed2c","remote_addr":"[fd00:1122:3344:101::c]:63779","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44248,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:38.980919875Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"c650020e-65ca-4652-8ac8-0f57a6cd5d8e","remote_addr":"[fd00:1122:3344:101::c]:63779","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":97,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:43.284037471Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55029"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:43.284770424Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"c23242de-6530-4240-a308-19b722b2e467","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55029","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":711,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:50.731941436Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:39459"}
{"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:50.743928583Z","hostname":"buskin","pid":1618,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:521","metadata":"InstanceMetadata { silo_id: e48373c8-b487-4cd1-adc6-a2c22d299e4c, project_id: 9bf892ed-1a58-4dcc-9342-5b6618092742 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-03-30T05:51:50.329873Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(b6608e45-b185-4760-b19c-f24aceaaafd2), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-30T05:51:50.355854109Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 57abfa80-97d5-4f38-b9f3-4d7ea6b36aa8, kind: Instance { id: daec6b21-c8aa-4afd-a556-e93742ed7c15 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 119, 207])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(16749977), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 57abfa80-97d5-4f38-b9f3-4d7ea6b36aa8, kind: Instance { id: daec6b21-c8aa-4afd-a556-e93742ed7c15 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 119, 207])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(16749977), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 57abfa80-97d5-4f38-b9f3-4d7ea6b36aa8, kind: Instance { id: daec6b21-c8aa-4afd-a556-e93742ed7c15 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 119, 207])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(16749977), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(16749977))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 57abfa80-97d5-4f38-b9f3-4d7ea6b36aa8, kind: Instance { id: daec6b21-c8aa-4afd-a556-e93742ed7c15 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 119, 207])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(16749977), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-f714748b0f3\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 221292c7-c1ce-449c-9a71-878f99132105, read_only_parent: Some(Volume { block_size: 512, id: 7210a1d9-d22d-49a7-87cd-02b628f37500, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 44eb95f9-0471-4a3f-af93-02e8ebe57571, key: Some(\"9pYb0mSs5cFWpa0MYvWpQcvVydm3Ueo3f/zh/alOoc4=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19002\", \"[fd00:1122:3344:101::18]:19001\", \"[fd00:1122:3344:101::15]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 221292c7-c1ce-449c-9a71-878f99132105, key: Some(\"SsgD9wq41gdRqKScyoI1RdX5djH0z6OYhPm0Sr76aJE=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::12]:19000\", \"[fd00:1122:3344:101::14]:19001\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAABEeS5+WH5YAAB5Ln5YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAARXkuflh+WAAAeS5+WAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJkYWVjNmIyMS1jOGFhLTRhZmQtYTU1Ni1lOTM3NDJlZDdjMTUiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJQmtvVFJpNndHQ0VUMHhyYXQ5RXoyeUtVZHdzdVhQUzZ1dWVhNk5CT3habCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"b6608e45-b185-4760-b19c-f24aceaaafd2","instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15"}
{"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:50.744036085Z","hostname":"buskin","pid":1618,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:555","instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15"}
{"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:50.744496413Z","hostname":"buskin","pid":1618,"component":"InstanceManager","file":"sled-agent/src/instance.rs:938","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 57abfa80-97d5-4f38-b9f3-4d7ea6b36aa8, kind: Instance { id: daec6b21-c8aa-4afd-a556-e93742ed7c15 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 119, 207])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(16749977), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 57abfa80-97d5-4f38-b9f3-4d7ea6b36aa8, kind: Instance { id: daec6b21-c8aa-4afd-a556-e93742ed7c15 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 119, 207])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(16749977), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 57abfa80-97d5-4f38-b9f3-4d7ea6b36aa8, kind: Instance { id: daec6b21-c8aa-4afd-a556-e93742ed7c15 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 119, 207])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(16749977), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(16749977))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 57abfa80-97d5-4f38-b9f3-4d7ea6b36aa8, kind: Instance { id: daec6b21-c8aa-4afd-a556-e93742ed7c15 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 119, 207])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(16749977), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-f714748b0f3\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 221292c7-c1ce-449c-9a71-878f99132105, read_only_parent: Some(Volume { block_size: 512, id: 7210a1d9-d22d-49a7-87cd-02b628f37500, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 44eb95f9-0471-4a3f-af93-02e8ebe57571, key: Some(\"9pYb0mSs5cFWpa0MYvWpQcvVydm3Ueo3f/zh/alOoc4=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19002\", \"[fd00:1122:3344:101::18]:19001\", \"[fd00:1122:3344:101::15]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 221292c7-c1ce-449c-9a71-878f99132105, key: Some(\"SsgD9wq41gdRqKScyoI1RdX5djH0z6OYhPm0Sr76aJE=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::12]:19000\", \"[fd00:1122:3344:101::14]:19001\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAABEeS5+WH5YAAB5Ln5YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAARXkuflh+WAAAeS5+WAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJkYWVjNmIyMS1jOGFhLTRhZmQtYTU1Ni1lOTM3NDJlZDdjMTUiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJQmtvVFJpNndHQ0VUMHhyYXQ5RXoyeUtVZHdzdVhQUzZ1dWVhNk5CT3habCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(b6608e45-b185-4760-b19c-f24aceaaafd2), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-30T05:51:50.355854109Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-03-30T05:51:50.329873Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"b6608e45-b185-4760-b19c-f24aceaaafd2","instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:50.745021033Z","hostname":"buskin","pid":1618,"uri":"/instances/daec6b21-c8aa-4afd-a556-e93742ed7c15","method":"PUT","req_id":"fbd154a7-4426-4260-b38c-4a7be595be84","remote_addr":"[fd00:1122:3344:101::a]:39459","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":13034,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:50.833940302Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:59534"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:50.835396489Z","hostname":"buskin","pid":1618,"uri":"/instances/daec6b21-c8aa-4afd-a556-e93742ed7c15/state","method":"PUT","req_id":"196ec3a0-18ba-4d4c-9046-fee6e8c08c00","remote_addr":"[fd00:1122:3344:101::a]:59534","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1372,"response_code":"200"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:50.859947033Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 250, 119, 207]), slot: 0, vni: Vni { inner: 16749977 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:50.860575994Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:50.894128292Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:50.914907068Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"system/illumos/propolis-server\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlInstance0\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::1:0\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"12400\"/>\n        <propval type=\"astring\" name=\"metric_addr\" value=\"[fd00:1122:3344:101::a]:12221\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:52.93656481Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:52.937006927Z","hostname":"buskin","pid":1618,"zone":"oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2","instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2\") failed: Property not found. retry in 60.49588ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:51:53.128183158Z","hostname":"buskin","pid":1618,"zone":"oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2","instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2\") failed: Property not found. retry in 124.09068ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:51:53.211578197Z","hostname":"buskin","pid":1618,"zone":"oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2","instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2\") failed: Property not found. retry in 174.781607ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:51:53.35535193Z","hostname":"buskin","pid":1618,"zone":"oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2","instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2\") failed: Property not found. retry in 267.855103ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:51:53.540857087Z","hostname":"buskin","pid":1618,"zone":"oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2","instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2\") failed: Property not found. retry in 859.613252ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:51:53.817053075Z","hostname":"buskin","pid":1618,"zone":"oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2","instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2\") failed: Property not found. retry in 1.403540649s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:51:54.688955077Z","hostname":"buskin","pid":1618,"zone":"oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2","instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2\") failed: Property not found. retry in 1.013682736s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:51:56.103350756Z","hostname":"buskin","pid":1618,"zone":"oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2","instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2\") failed: Property not found. retry in 615.82741ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:51:57.127020775Z","hostname":"buskin","pid":1618,"zone":"oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2","instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Started propolis in zone: oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:57.773836451Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"sled-agent/src/instance.rs:1423"}
{"msg":"wait for service Some(\"oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2\") failed: Property not found. retry in 44.676269ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:51:57.78315667Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2\") failed: Property not found. retry in 106.480349ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:51:57.840805463Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2\") failed: Property not found. retry in 229.802988ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:51:57.957933323Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2\") failed: Property not found. retry in 583.68864ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:51:58.197731064Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:58.79144479Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"sled-agent/src/instance.rs:1432"}
{"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:58.82738176Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"sled-agent/src/instance.rs:1446"}
{"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAABEeS5+WH5YAAB5Ln5YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAARXkuflh+WAAAeS5+WAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJkYWVjNmIyMS1jOGFhLTRhZmQtYTU1Ni1lOTM3NDJlZDdjMTUiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJQmtvVFJpNndHQ0VUMHhyYXQ5RXoyeUtVZHdzdVhQUzZ1dWVhNk5CT3habCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-f714748b0f3\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 221292c7-c1ce-449c-9a71-878f99132105, read_only_parent: Some(Volume { block_size: 512, id: 7210a1d9-d22d-49a7-87cd-02b628f37500, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 44eb95f9-0471-4a3f-af93-02e8ebe57571, key: Some(\"9pYb0mSs5cFWpa0MYvWpQcvVydm3Ueo3f/zh/alOoc4=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19002\", \"[fd00:1122:3344:101::18]:19001\", \"[fd00:1122:3344:101::15]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 221292c7-c1ce-449c-9a71-878f99132105, key: Some(\"SsgD9wq41gdRqKScyoI1RdX5djH0z6OYhPm0Sr76aJE=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::12]:19000\", \"[fd00:1122:3344:101::14]:19001\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: daec6b21-c8aa-4afd-a556-e93742ed7c15, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: 9bf892ed-1a58-4dcc-9342-5b6618092742, silo_id: e48373c8-b487-4cd1-adc6-a2c22d299e4c }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:58.833066013Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"sled-agent/src/instance.rs:685"}
{"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:58.998191982Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"sled-agent/src/instance.rs:687"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-03-30T05:51:58.998625290Z }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:58.99865957Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:58.998696921Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-03-30T05:51:58.998625290Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(b6608e45-b185-4760-b19c-f24aceaaafd2), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-30T05:51:50.355854109Z }","propolis_id":"b6608e45-b185-4760-b19c-f24aceaaafd2"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:58.998720341Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(b6608e45-b185-4760-b19c-f24aceaaafd2), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-30T05:51:50.355854109Z }, propolis_id: b6608e45-b185-4760-b19c-f24aceaaafd2, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-03-30T05:51:58.998625290Z } }","instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-03-30T05:51:59.058814279Z }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:59.05884038Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:59.05885541Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-03-30T05:51:59.058814279Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(b6608e45-b185-4760-b19c-f24aceaaafd2), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-30T05:51:50.355854109Z }","propolis_id":"b6608e45-b185-4760-b19c-f24aceaaafd2"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:51:59.05886433Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(b6608e45-b185-4760-b19c-f24aceaaafd2), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-30T05:51:50.355854109Z }, propolis_id: b6608e45-b185-4760-b19c-f24aceaaafd2, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-03-30T05:51:59.058814279Z } }","instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:00.611799267Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/93b0cc91-dc62-481c-9ea1-425d29f058ae","method":"GET","req_id":"5b3c2898-85ff-41ff-b20e-3b016e086931","remote_addr":"[fd00:1122:3344:101::d]:56676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":558,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:07.571619218Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:40666"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:07.572325831Z","hostname":"buskin","pid":1618,"uri":"/instances/daec6b21-c8aa-4afd-a556-e93742ed7c15/state","method":"PUT","req_id":"7ca76ef8-dc4c-4684-83a9-41dce19229aa","remote_addr":"[fd00:1122:3344:101::a]:40666","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":626,"response_code":"200"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-03-30T05:52:07.572484614Z }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:07.572515854Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:07.572561225Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-03-30T05:52:07.572484614Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(b6608e45-b185-4760-b19c-f24aceaaafd2), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-30T05:51:50.355854109Z }","propolis_id":"b6608e45-b185-4760-b19c-f24aceaaafd2"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:07.572582556Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(b6608e45-b185-4760-b19c-f24aceaaafd2), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-30T05:51:50.355854109Z }, propolis_id: b6608e45-b185-4760-b19c-f24aceaaafd2, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-03-30T05:52:07.572484614Z } }","instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-03-30T05:52:07.594394020Z }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:07.59441096Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:07.594427851Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-03-30T05:52:07.594394020Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-03-30T05:52:07.594394020Z }","propolis_id":"b6608e45-b185-4760-b19c-f24aceaaafd2"}
{"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:07.594437941Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:07.595029502Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:07.597434955Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:07.597473276Z","hostname":"buskin","pid":1618,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:437","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/eaeb58f9-7756-4551-8a4b-dc00787e5906/debug/bundle/zone\", \"/pool/int/f076b946-15f4-4270-9a34-aa6e75797fbb/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/82347231-68ab-4934-8007-b28d9a3a3777/crypt/debug\", \"/pool/ext/28899eb6-914e-44b6-b77e-9d82d92186d6/crypt/debug\", \"/pool/ext/6de8542c-123a-4abc-9a88-e72f1063a9f4/crypt/debug\", \"/pool/ext/e43353e3-7e66-4e6f-bd56-19f8143ee8b5/crypt/debug\", \"/pool/ext/5c3019ca-328c-47ab-9982-06e092199db8/crypt/debug\", \"/pool/ext/3c92b6bd-8180-45a3-9123-a99ddc649159/crypt/debug\", \"/pool/ext/ea19d3d4-e058-46fb-981b-9ccb255f563c/crypt/debug\", \"/pool/ext/e22df0ad-bbf8-4902-93d6-e67ff23c3234/crypt/debug\", \"/pool/ext/c7e0e489-6c2b-472e-8a1f-efe83bb548dd/crypt/debug\"] }","zone_name":"oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2"}
{"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:08.197622508Z","hostname":"buskin","pid":1618,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1236","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2\", bundle_id: 2a5eb9ef-7a71-4103-b991-5f0a32187f1d }, time_created: 2024-03-30T05:52:07.598294710Z, version: 0, cause: TerminatedInstance }"}
{"msg":"Halting and removing zone: oxz_propolis-server_b6608e45-b185-4760-b19c-f24aceaaafd2","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:52:08.19771002Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:09.069072922Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:42902"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:09.112237783Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"0f55b517-5a9c-4f00-a1bf-044fb3962297","remote_addr":"[fd00:1122:3344:101::b]:42902","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43062,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:09.112540728Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"b8cd97ee-82f5-4e15-96fd-896311144a11","remote_addr":"[fd00:1122:3344:101::b]:42902","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":121,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:09.191821343Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:46068"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:09.235929852Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"bb297b8a-3ff6-40f3-8bf0-19620fede39b","remote_addr":"[fd00:1122:3344:101::c]:46068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44030,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:09.236981331Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"30fe8926-0533-4cb7-ad20-fcc56fd8b59c","remote_addr":"[fd00:1122:3344:101::c]:46068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":885,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:09.293075406Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:47720"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:09.342562232Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"9861bf28-e308-4368-8dd1-dad3742ab7fd","remote_addr":"[fd00:1122:3344:101::a]:47720","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49408,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:09.342807536Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"791b2f8a-151d-4341-8099-c189e4a3507c","remote_addr":"[fd00:1122:3344:101::a]:47720","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":90,"response_code":"200"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:09.948131722Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:09.972529303Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-03-30T05:52:07.594394020Z }, propolis_id: b6608e45-b185-4760-b19c-f24aceaaafd2, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-03-30T05:52:09.972458882Z } }","instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:10.049440786Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:57877"}
{"msg":"unset_virtual_nic_host unimplmented","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T05:52:10.054069889Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:620"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:10.0541022Z","hostname":"buskin","pid":1618,"uri":"/v2p/57abfa80-97d5-4f38-b9f3-4d7ea6b36aa8","method":"DELETE","req_id":"397a391e-6153-4fda-b162-819fd36e4271","remote_addr":"[fd00:1122:3344:101::c]:57877","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":2530,"response_code":"204"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:10.282042425Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-03-30T05:52:07.594394020Z }, propolis_id: b6608e45-b185-4760-b19c-f24aceaaafd2, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-03-30T05:52:09.972458882Z } }","instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15"}
{"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T05:52:10.284906617Z","hostname":"buskin","pid":1618,"instance_id":"daec6b21-c8aa-4afd-a556-e93742ed7c15","component":"InstanceManager","file":"sled-agent/src/instance.rs:723"}